test(e2e): restore webhook redelivery regression coverage (#5690)

test(e2e): unquarantine webhook redelivery regression

crates/e2e_test/src/notification_webhook_test.rs

@@ -24,7 +24,7 @@
 //!   * PUT / multipart-complete / DeleteObject / DeleteObjects each deliver one event with the correct
 //!     eventName, bucket, key, versionId and eTag.
 //!   * prefix/suffix filters drop non-matching keys (rule-engine gate).
-//!   * an event queued while the target endpoint is unreachable is redelivered
+//!   * an event queued while the target endpoint rejects delivery is redelivered
 //!     from the on-disk store once the endpoint recovers (store-and-forward).
 //!   * responseElements and the S3 response use the canonical request ID while
 //!     requestParameters preserve a conflicting client-supplied value.
@@ -897,11 +897,10 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
     Ok(())
 }
 
-/// An event queued while the target endpoint is unreachable survives on the
+/// An event queued while the target endpoint rejects delivery survives on the
 /// durable store and is redelivered once the endpoint comes back.
 #[tokio::test]
 #[serial]
-#[ignore = "FAILING deterministically on main since it landed (#4821): the target is created but never appears in /rustfs/admin/v3/target/arns, so wait_for_target_registered times out. Quarantined per the flake policy; remove with the fix for rustfs#4852"]
 async fn test_webhook_redelivers_event_after_target_recovers() -> TestResult {
     init_logging();
 
@@ -932,28 +931,55 @@ async fn test_webhook_redelivers_event_after_target_recovers() -> TestResult {
     wait_for_target_registered(&env, target).await?;
     put_notification_config(&client, bucket, target, "uploads/", ".dat").await?;
 
-    // Take the endpoint down (drops the listener, so connections are refused —
-    // a retryable NotConnected), then PUT: the event cannot be delivered and
-    // must survive on the durable queue store.
+    // Replace the healthy setup listener with one that rejects the first POST.
+    // Waiting for that response below proves the queued event reached a failed
+    // delivery attempt before the endpoint recovers.
     setup_handle.abort();
     let _ = setup_handle.await;
 
+    let listener = TcpListener::bind(("0.0.0.0", port)).await?;
     let key = "uploads/redeliver.dat";
     client
         .put_object()
         .bucket(bucket)
         .key(key)
-        .body(ByteStream::from_static(b"queued while target down"))
+        .body(ByteStream::from_static(b"queued while target rejects"))
         .send()
         .await?;
 
-    // Hold the endpoint down long enough for at least one replay attempt to
-    // fail (the replay worker scans the store every 500ms), so recovery below
-    // exercises real redelivery rather than a first-attempt success.
-    tokio::time::sleep(Duration::from_secs(2)).await;
+    let mut failure_handle = tokio::spawn(async move {
+        loop {
+            let (mut stream, _) = listener.accept().await?;
+            let (method, _) = timeout(Duration::from_secs(5), read_http_message(&mut stream)).await??;
+            if method == "HEAD" {
+                stream
+                    .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
+                    .await?;
+                stream.shutdown().await?;
+                continue;
+            }
+            if method == "POST" {
+                stream
+                    .write_all(b"HTTP/1.1 503 Service Unavailable\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
+                    .await?;
+                stream.shutdown().await?;
+                return Ok::<(), BoxError>(());
+            }
+        }
+    });
 
-    // Bring the endpoint back on the same port; the replay worker retries with
-    // exponential backoff and delivers the queued event.
+    let rejected = match timeout(Duration::from_secs(20), &mut failure_handle).await {
+        Ok(rejected) => rejected,
+        Err(_) => {
+            failure_handle.abort();
+            let _ = failure_handle.await;
+            return Err("webhook replay did not reach the rejecting endpoint".into());
+        }
+    };
+    rejected??;
+
+    // Bring the endpoint back on the same port; the replay worker rescans the
+    // durable queue and delivers the retained event.
     let listener = TcpListener::bind(("0.0.0.0", port)).await?;
     let (tx, mut rx) = mpsc::unbounded_channel();
     let handle = serve_event_collector(listener, tx);

docs/testing/e2e-suite-inventory.md

@@ -66,7 +66,7 @@
 | multipart_storage_class_test | 3 | ✅ |
 | namespace_lock_quorum_test | 2 |  |
 | negative_sigv4_test | 6 | ✅ |
-| notification_webhook_test | 2 | ✅ |
+| notification_webhook_test | 3 | ✅ |
 | object_lambda_test | 16 |  |
 | object_lock | 33 |  |
 | overwrite_cleanup_regression_test | 1 |  |
@@ -88,6 +88,4 @@
 | tls_hot_reload_test | 1 | ✅ |
 | version_id_regression_test | 10 | ✅ |
 
-`notification_webhook_test` also has 1 ignored store-and-forward regression tracked by rustfs#4852; ignored tests are excluded from the active counts above.
-
-**Total listed: 527 tests across 70 modules · PR smoke subset: 147 tests / 33 modules** (31 full modules + 18 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 28 tests** · generated 2026-07-30.
+**Total listed: 528 tests across 70 modules · PR smoke subset: 148 tests / 33 modules** (31 full modules + 18 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 28 tests** · generated 2026-08-04.