mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-27 00:38:16 +00:00
Zhengchao An
cf142e7fdd
Follow-up to #4772. After forcing every-cycle ILM evaluation the Days=1 plateau is reliable, but test_lifecycle_expiration / test_lifecyclev2_expiration still flaked intermittently on the *second* assertion (`assert 4 == 2`): the Days=5 (expire3) objects were not expired within their poll window. Cause: _wait_for_lifecycle_count for expire3 starts its N*lc_interval deadline only after the Days=1 poll returns (~1 debug-day in). With N=5 and lc_interval == debug_day, the 5*debug_day terms cancel and the slack past the Days=5 due time is only ~1 debug-day (~9s) -- which a single slow scanner cycle (observed ~13s spacing under CI load) can exceed, leaving the count stalled at 4. Bumping the two expire3 windows to 8*lc_interval raises the slack to ~39s, comfortably above the observed cycle jitter. Test-only, lane-scoped: does not touch RUSTFS_ILM_DEBUG_DAY_SECS or the 4*lc_interval < 5*debug_day plateau invariant. The expire3 target count (2) is terminal (nothing expires after it), so a wider window can never over-expire. Also documents the #4772 RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=1 knob in lifecycle_behavior_tests.txt.
141 lines
5.4 KiB
Diff
141 lines
5.4 KiB
Diff
diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py
|
|
index bf2f2b1..f3ef394 100644
|
|
--- a/s3tests/functional/test_s3.py
|
|
+++ b/s3tests/functional/test_s3.py
|
|
@@ -8509,6 +8509,26 @@ def test_lifecycle_get_no_id():
|
|
print("rules not right")
|
|
assert False
|
|
|
|
+def _wait_for_lifecycle_count(fetch_items, expected_count, timeout):
|
|
+ deadline = time.monotonic() + timeout
|
|
+ items = fetch_items()
|
|
+ while len(items) > expected_count:
|
|
+ remaining = deadline - time.monotonic()
|
|
+ if remaining <= 0:
|
|
+ break
|
|
+ time.sleep(min(1, remaining))
|
|
+ items = fetch_items()
|
|
+ return items
|
|
+
|
|
+
|
|
+def _list_multipart_uploads_for_keys(client, bucket_name, key_names):
|
|
+ uploads = []
|
|
+ for key in key_names:
|
|
+ response = client.list_multipart_uploads(Bucket=bucket_name, Prefix=key)
|
|
+ uploads.extend(upload for upload in response.get('Uploads', [])
|
|
+ if upload['Key'] == key)
|
|
+ return uploads
|
|
+
|
|
+
|
|
-# The test harness for lifecycle is configured to treat days as 10 second intervals.
|
|
@pytest.mark.lifecycle
|
|
@pytest.mark.lifecycle_expiration
|
|
@@ -8527,17 +8548,13 @@ def test_lifecycle_expiration():
|
|
|
|
lc_interval = get_lc_debug_interval()
|
|
|
|
- time.sleep(3*lc_interval)
|
|
- response = client.list_objects(Bucket=bucket_name)
|
|
- expire1_objects = response['Contents']
|
|
-
|
|
- time.sleep(lc_interval)
|
|
- response = client.list_objects(Bucket=bucket_name)
|
|
- keep2_objects = response['Contents']
|
|
-
|
|
- time.sleep(3*lc_interval)
|
|
- response = client.list_objects(Bucket=bucket_name)
|
|
- expire3_objects = response['Contents']
|
|
+ expire1_objects = _wait_for_lifecycle_count(
|
|
+ lambda: client.list_objects(Bucket=bucket_name).get('Contents', []),
|
|
+ 4, 4*lc_interval)
|
|
+ keep2_objects = client.list_objects(Bucket=bucket_name).get('Contents', [])
|
|
+ expire3_objects = _wait_for_lifecycle_count(
|
|
+ lambda: client.list_objects(Bucket=bucket_name).get('Contents', []),
|
|
+ 2, 8*lc_interval)
|
|
|
|
assert len(init_objects) == 6
|
|
assert len(expire1_objects) == 4
|
|
@@ -8562,17 +8579,13 @@ def test_lifecyclev2_expiration():
|
|
|
|
lc_interval = get_lc_debug_interval()
|
|
|
|
- time.sleep(3*lc_interval)
|
|
- response = client.list_objects_v2(Bucket=bucket_name)
|
|
- expire1_objects = response['Contents']
|
|
-
|
|
- time.sleep(lc_interval)
|
|
- response = client.list_objects_v2(Bucket=bucket_name)
|
|
- keep2_objects = response['Contents']
|
|
-
|
|
- time.sleep(3*lc_interval)
|
|
- response = client.list_objects_v2(Bucket=bucket_name)
|
|
- expire3_objects = response['Contents']
|
|
+ expire1_objects = _wait_for_lifecycle_count(
|
|
+ lambda: client.list_objects_v2(Bucket=bucket_name).get('Contents', []),
|
|
+ 4, 4*lc_interval)
|
|
+ keep2_objects = client.list_objects_v2(Bucket=bucket_name).get('Contents', [])
|
|
+ expire3_objects = _wait_for_lifecycle_count(
|
|
+ lambda: client.list_objects_v2(Bucket=bucket_name).get('Contents', []),
|
|
+ 2, 8*lc_interval)
|
|
|
|
assert len(init_objects) == 6
|
|
assert len(expire1_objects) == 4
|
|
@@ -9308,11 +9321,9 @@ def test_lifecycle_noncur_expiration():
|
|
|
|
lc_interval = get_lc_debug_interval()
|
|
|
|
- # Wait for first expiration (plus fudge to handle the timer window)
|
|
- time.sleep(5*lc_interval)
|
|
-
|
|
- response = client.list_object_versions(Bucket=bucket_name)
|
|
- expire_versions = response['Versions']
|
|
+ expire_versions = _wait_for_lifecycle_count(
|
|
+ lambda: client.list_object_versions(Bucket=bucket_name).get('Versions', []),
|
|
+ 4, 6*lc_interval)
|
|
assert len(init_versions) == 6
|
|
assert len(expire_versions) == 4
|
|
|
|
@@ -9367,13 +9378,12 @@ def test_lifecycle_deletemarker_expiration():
|
|
|
|
lc_interval = get_lc_debug_interval()
|
|
|
|
- # Wait for first expiration (plus fudge to handle the timer window)
|
|
- time.sleep(7*lc_interval)
|
|
+ def current_versions():
|
|
+ response = client.list_object_versions(Bucket=bucket_name)
|
|
+ return response.get('Versions', []) + response.get('DeleteMarkers', [])
|
|
|
|
- response = client.list_object_versions(Bucket=bucket_name)
|
|
- init_versions = response['Versions']
|
|
- deleted_versions = response['DeleteMarkers']
|
|
- total_expire_versions = init_versions + deleted_versions
|
|
+ total_expire_versions = _wait_for_lifecycle_count(
|
|
+ current_versions, 2, 8*lc_interval)
|
|
|
|
assert len(total_init_versions) == 4
|
|
assert len(total_expire_versions) == 2
|
|
@@ -9441,8 +9451,7 @@ def test_lifecycle_multipart_expiration():
|
|
response = client.create_multipart_upload(Bucket=bucket_name, Key=key)
|
|
upload_ids.append(response['UploadId'])
|
|
|
|
- response = client.list_multipart_uploads(Bucket=bucket_name)
|
|
- init_uploads = response['Uploads']
|
|
+ init_uploads = _list_multipart_uploads_for_keys(client, bucket_name, key_names)
|
|
|
|
rules = [
|
|
{'ID': 'rule1', 'Prefix': 'test1/', 'Status': 'Enabled',
|
|
@@ -9453,10 +9462,8 @@ def test_lifecycle_multipart_expiration():
|
|
|
|
lc_interval = get_lc_debug_interval()
|
|
|
|
- # Wait for first expiration (plus fudge to handle the timer window)
|
|
- time.sleep(5*lc_interval)
|
|
-
|
|
- response = client.list_multipart_uploads(Bucket=bucket_name)
|
|
- expired_uploads = response['Uploads']
|
|
+ expired_uploads = _wait_for_lifecycle_count(
|
|
+ lambda: _list_multipart_uploads_for_keys(client, bucket_name, key_names),
|
|
+ 1, 6*lc_interval)
|
|
assert len(init_uploads) == 2
|
|
assert len(expired_uploads) == 1
|