fix: stabilize s3-tests delete key-limit coverage (#4283)

* fix: tighten list handling and s3 test support

* chore: tidy imports and metric updates

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
Signed-off-by: Zhengchao An <anzhengchao@gmail.com>

* fix: address s3tests review follow-ups

---------

Signed-off-by: Zhengchao An <anzhengchao@gmail.com>
Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

scripts/s3-tests/patches/0001-rustfs-key-limit-timeout-and-utcnow.patch

@@ -0,0 +1,75 @@
+diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py
+index 2ee91b3..bf2f2b1 100644
+--- a/s3tests/functional/test_s3.py
++++ b/s3tests/functional/test_s3.py
+@@ -138,6 +138,9 @@ def _create_objects(bucket=None, bucket_name=None, keys=[], put_object_args={}):
+ 
+     return bucket_name
+ 
++def _utcnow_naive():
++    return datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
++
+ def _get_keys(response):
+     """
+     return lists of strings that are the keys from a client.list_objects() response
+@@ -1760,15 +1763,8 @@ def test_multi_objectv2_delete():
+ 
+ def test_multi_object_delete_key_limit():
+     key_names = [f"key-{i}" for i in range(1001)]
+-    bucket_name = _create_objects(keys=key_names)
+     client = get_client()
+-
+-    paginator = client.get_paginator('list_objects')
+-    pages = paginator.paginate(Bucket=bucket_name)
+-    numKeys = 0
+-    for page in pages:
+-        numKeys += len(page['Contents'])
+-    assert numKeys == 1001
++    bucket_name = get_new_bucket(client)
+ 
+     objs_dict = _make_objs_dict(key_names=key_names)
+     e = assert_raises(ClientError,client.delete_objects,Bucket=bucket_name,Delete=objs_dict)
+@@ -1777,15 +1773,8 @@ def test_multi_object_delete_key_limit():
+ 
+ def test_multi_objectv2_delete_key_limit():
+     key_names = [f"key-{i}" for i in range(1001)]
+-    bucket_name = _create_objects(keys=key_names)
+     client = get_client()
+-
+-    paginator = client.get_paginator('list_objects_v2')
+-    pages = paginator.paginate(Bucket=bucket_name)
+-    numKeys = 0
+-    for page in pages:
+-        numKeys += len(page['Contents'])
+-    assert numKeys == 1001
++    bucket_name = get_new_bucket(client)
+ 
+     objs_dict = _make_objs_dict(key_names=key_names)
+     e = assert_raises(ClientError,client.delete_objects,Bucket=bucket_name,Delete=objs_dict)
+@@ -9163,7 +9152,7 @@ def test_lifecycle_expiration_header_put():
+     bucket_name = get_new_bucket()
+     client = get_client()
+ 
+-    now = datetime.datetime.utcnow()
++    now = _utcnow_naive()
+     response = setup_lifecycle_expiration(
+         client, bucket_name, 'rule1', 1, 'days1/')
+     assert check_lifecycle_expiration_header(response, now, 'rule1', 1)
+@@ -9175,7 +9164,7 @@ def test_lifecycle_expiration_header_head():
+     bucket_name = get_new_bucket()
+     client = get_client()
+ 
+-    now = datetime.datetime.utcnow()
++    now = _utcnow_naive()
+     response = setup_lifecycle_expiration(
+         client, bucket_name, 'rule1', 1, 'days1/')
+ 
+@@ -9246,7 +9235,7 @@ def test_lifecycle_expiration_header_tags_head():
+ @pytest.mark.lifecycle_expiration
+ @pytest.mark.fails_on_dbstore
+ def test_lifecycle_expiration_header_and_tags_head():
+-    now = datetime.datetime.utcnow()
++    now = _utcnow_naive()
+     bucket_name = get_new_bucket()
+     client = get_client()
+     lifecycle={

scripts/s3-tests/run.sh

@@ -96,6 +96,55 @@ log_error() {
     echo -e "${RED}[ERROR]${NC} $*"
 }
 
+summarize_junit_failures() {
+    local junit_path="$1"
+
+    if [ ! -f "${junit_path}" ]; then
+        log_warn "JUnit report not found: ${junit_path}"
+        return 0
+    fi
+
+    python3 - "${junit_path}" <<'PY'
+import sys
+import xml.etree.ElementTree as ET
+
+junit_path = sys.argv[1]
+try:
+    root = ET.parse(junit_path).getroot()
+except Exception as exc:
+    print(f"[WARN] Failed to parse JUnit report {junit_path}: {exc}")
+    raise SystemExit(0)
+
+failures = []
+for case in root.iter("testcase"):
+    failure = case.find("failure")
+    error = case.find("error")
+    node = failure if failure is not None else error
+    if node is None:
+        continue
+
+    classname = case.attrib.get("classname", "")
+    name = case.attrib.get("name", "")
+    duration = case.attrib.get("time", "0")
+    message = node.attrib.get("message") or (node.text or "").strip().splitlines()[0:1]
+    if isinstance(message, list):
+        message = message[0] if message else ""
+    failures.append((classname, name, duration, message))
+
+if not failures:
+    print("[INFO] No failed testcases found in JUnit report")
+    raise SystemExit(0)
+
+print("[ERROR] s3-tests failed testcase summary:")
+for classname, name, duration, message in failures[:20]:
+    nodeid = f"{classname}::{name}" if classname else name
+    print(f"[ERROR] - {nodeid} ({duration}s): {message}")
+
+if len(failures) > 20:
+    print(f"[ERROR] - ... {len(failures) - 20} additional failed testcases omitted")
+PY
+}
+
 # =============================================================================
 # Test Classification Files
 # =============================================================================
@@ -898,6 +947,18 @@ git -C "${PROJECT_ROOT}/s3-tests" checkout -qf --detach "${S3TESTS_REV}" || {
     exit 1
 }
 
+S3TESTS_PATCH_DIR="${SCRIPT_DIR}/patches"
+if [ -d "${S3TESTS_PATCH_DIR}" ]; then
+    for patch_file in "${S3TESTS_PATCH_DIR}"/*.patch; do
+        [ -e "${patch_file}" ] || continue
+        log_info "Applying s3-tests patch: ${patch_file##*/}"
+        git -C "${PROJECT_ROOT}/s3-tests" apply "${patch_file}" || {
+            log_error "Failed to apply s3-tests patch: ${patch_file}"
+            exit 1
+        }
+    done
+fi
+
 cd "${PROJECT_ROOT}/s3-tests"
 
 # Install tox if not available
@@ -940,6 +1001,7 @@ else
 fi
 
 # Run tests from s3tests/functional
+set +e
 S3TEST_CONF="${CONF_OUTPUT_PATH}" \
     tox -- \
     -vv -ra --showlocals --tb=long \
@@ -951,6 +1013,7 @@ S3TEST_CONF="${CONF_OUTPUT_PATH}" \
     2>&1 | tee "${ARTIFACTS_DIR}/pytest.log"
 
 TEST_EXIT_CODE=${PIPESTATUS[0]}
+set -e
 
 # Step 10: Collect RustFS logs
 log_info "Collecting RustFS logs..."
@@ -977,6 +1040,10 @@ if [ -f "${REPORT_SCRIPT}" ] && [ -f "${ARTIFACTS_DIR}/junit.xml" ]; then
         || log_warn "Compatibility report generation failed"
 fi
 
+if [ ${TEST_EXIT_CODE} -ne 0 ]; then
+    summarize_junit_failures "${ARTIFACTS_DIR}/junit.xml"
+fi
+
 # Summary
 if [ ${TEST_EXIT_CODE} -eq 0 ]; then
     log_info "Tests completed successfully!"