From cf142e7fddb938cea2d27dd16a3ba68d44263303 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 12 Jul 2026 22:15:05 +0800 Subject: [PATCH] fix(ci): widen Days=5 expiration poll window to 8*lc_interval (#4779) Follow-up to #4772. After forcing every-cycle ILM evaluation the Days=1 plateau is reliable, but test_lifecycle_expiration / test_lifecyclev2_expiration still flaked intermittently on the *second* assertion (`assert 4 == 2`): the Days=5 (expire3) objects were not expired within their poll window. Cause: _wait_for_lifecycle_count for expire3 starts its N*lc_interval deadline only after the Days=1 poll returns (~1 debug-day in). With N=5 and lc_interval == debug_day, the 5*debug_day terms cancel and the slack past the Days=5 due time is only ~1 debug-day (~9s) -- which a single slow scanner cycle (observed ~13s spacing under CI load) can exceed, leaving the count stalled at 4. Bumping the two expire3 windows to 8*lc_interval raises the slack to ~39s, comfortably above the observed cycle jitter. Test-only, lane-scoped: does not touch RUSTFS_ILM_DEBUG_DAY_SECS or the 4*lc_interval < 5*debug_day plateau invariant. The expire3 target count (2) is terminal (nothing expires after it), so a wider window can never over-expire. Also documents the #4772 RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=1 knob in lifecycle_behavior_tests.txt. --- scripts/s3-tests/lifecycle_behavior_tests.txt | 13 +++++++++++++ .../patches/0002-lifecycle-behavior-polling.patch | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/s3-tests/lifecycle_behavior_tests.txt b/scripts/s3-tests/lifecycle_behavior_tests.txt index 6d0cac31d..513da91e8 100644 --- a/scripts/s3-tests/lifecycle_behavior_tests.txt +++ b/scripts/s3-tests/lifecycle_behavior_tests.txt @@ -20,9 +20,22 @@ # RUSTFS_ILM_DEBUG_DAY_SECS=10 (1 "day" = 10s, matches s3-tests lc_debug_interval) # RUSTFS_SCANNER_ENABLED=true # RUSTFS_SCANNER_CYCLE=2 (scan every 2s, well under a debug-day) +# RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=1 (re-descend compacted dirs every cycle so +# ILM is evaluated ~every 2s, not once per 16 cycles; +# without this a Days=1 object due at 10s is not +# expired until the next ~32s boundary — #4772) # RUSTFS_SCANNER_START_DELAY_SECS=0 # RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL=2s (multipart abort lane) # +# Polling windows (patches/0002): each assertion polls list_objects until the +# expected count is reached or a N*lc_interval deadline elapses. The Days=5 +# (expire3) window uses 8*lc_interval rather than 5: its deadline only starts +# after the Days=1 poll returns (~1 debug-day in), so a 5*lc_interval window left +# only ~1 debug-day of slack past the Days=5 due time — a single slow scanner +# cycle (observed ~13s under CI load) could push the expiry just past it and +# stall the count at 4 (assert 4 == 2). 8*lc_interval restores comfortable slack; +# the target count (2) is terminal so a wider window never over-expires. +# # Point run.sh at this file with IMPLEMENTED_TESTS_FILE=. # # First batch (ilm-10). Each entry was verified against the pinned upstream source diff --git a/scripts/s3-tests/patches/0002-lifecycle-behavior-polling.patch b/scripts/s3-tests/patches/0002-lifecycle-behavior-polling.patch index abeabee97..3f9354490 100644 --- a/scripts/s3-tests/patches/0002-lifecycle-behavior-polling.patch +++ b/scripts/s3-tests/patches/0002-lifecycle-behavior-polling.patch @@ -51,7 +51,7 @@ index bf2f2b1..f3ef394 100644 + keep2_objects = client.list_objects(Bucket=bucket_name).get('Contents', []) + expire3_objects = _wait_for_lifecycle_count( + lambda: client.list_objects(Bucket=bucket_name).get('Contents', []), -+ 2, 5*lc_interval) ++ 2, 8*lc_interval) assert len(init_objects) == 6 assert len(expire1_objects) == 4 @@ -76,7 +76,7 @@ index bf2f2b1..f3ef394 100644 + keep2_objects = client.list_objects_v2(Bucket=bucket_name).get('Contents', []) + expire3_objects = _wait_for_lifecycle_count( + lambda: client.list_objects_v2(Bucket=bucket_name).get('Contents', []), -+ 2, 5*lc_interval) ++ 2, 8*lc_interval) assert len(init_objects) == 6 assert len(expire1_objects) == 4