diff --git a/crates/e2e_test/src/notification_webhook_test.rs b/crates/e2e_test/src/notification_webhook_test.rs index 86c546e82..e63cea554 100644 --- a/crates/e2e_test/src/notification_webhook_test.rs +++ b/crates/e2e_test/src/notification_webhook_test.rs @@ -24,7 +24,7 @@ //! * PUT / multipart-complete / DeleteObject / DeleteObjects each deliver one event with the correct //! eventName, bucket, key, versionId and eTag. //! * prefix/suffix filters drop non-matching keys (rule-engine gate). -//! * an event queued while the target endpoint is unreachable is redelivered +//! * an event queued while the target endpoint rejects delivery is redelivered //! from the on-disk store once the endpoint recovers (store-and-forward). //! * responseElements and the S3 response use the canonical request ID while //! requestParameters preserve a conflicting client-supplied value. @@ -897,11 +897,10 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult { Ok(()) } -/// An event queued while the target endpoint is unreachable survives on the +/// An event queued while the target endpoint rejects delivery survives on the /// durable store and is redelivered once the endpoint comes back. #[tokio::test] #[serial] -#[ignore = "FAILING deterministically on main since it landed (#4821): the target is created but never appears in /rustfs/admin/v3/target/arns, so wait_for_target_registered times out. Quarantined per the flake policy; remove with the fix for rustfs#4852"] async fn test_webhook_redelivers_event_after_target_recovers() -> TestResult { init_logging(); @@ -932,28 +931,55 @@ async fn test_webhook_redelivers_event_after_target_recovers() -> TestResult { wait_for_target_registered(&env, target).await?; put_notification_config(&client, bucket, target, "uploads/", ".dat").await?; - // Take the endpoint down (drops the listener, so connections are refused — - // a retryable NotConnected), then PUT: the event cannot be delivered and - // must survive on the durable queue store. + // Replace the healthy setup listener with one that rejects the first POST. + // Waiting for that response below proves the queued event reached a failed + // delivery attempt before the endpoint recovers. setup_handle.abort(); let _ = setup_handle.await; + let listener = TcpListener::bind(("0.0.0.0", port)).await?; let key = "uploads/redeliver.dat"; client .put_object() .bucket(bucket) .key(key) - .body(ByteStream::from_static(b"queued while target down")) + .body(ByteStream::from_static(b"queued while target rejects")) .send() .await?; - // Hold the endpoint down long enough for at least one replay attempt to - // fail (the replay worker scans the store every 500ms), so recovery below - // exercises real redelivery rather than a first-attempt success. - tokio::time::sleep(Duration::from_secs(2)).await; + let mut failure_handle = tokio::spawn(async move { + loop { + let (mut stream, _) = listener.accept().await?; + let (method, _) = timeout(Duration::from_secs(5), read_http_message(&mut stream)).await??; + if method == "HEAD" { + stream + .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n") + .await?; + stream.shutdown().await?; + continue; + } + if method == "POST" { + stream + .write_all(b"HTTP/1.1 503 Service Unavailable\r\ncontent-length: 0\r\nconnection: close\r\n\r\n") + .await?; + stream.shutdown().await?; + return Ok::<(), BoxError>(()); + } + } + }); - // Bring the endpoint back on the same port; the replay worker retries with - // exponential backoff and delivers the queued event. + let rejected = match timeout(Duration::from_secs(20), &mut failure_handle).await { + Ok(rejected) => rejected, + Err(_) => { + failure_handle.abort(); + let _ = failure_handle.await; + return Err("webhook replay did not reach the rejecting endpoint".into()); + } + }; + rejected??; + + // Bring the endpoint back on the same port; the replay worker rescans the + // durable queue and delivers the retained event. let listener = TcpListener::bind(("0.0.0.0", port)).await?; let (tx, mut rx) = mpsc::unbounded_channel(); let handle = serve_event_collector(listener, tx); diff --git a/docs/testing/e2e-suite-inventory.md b/docs/testing/e2e-suite-inventory.md index 8b2192e35..c6dcb30b3 100644 --- a/docs/testing/e2e-suite-inventory.md +++ b/docs/testing/e2e-suite-inventory.md @@ -66,7 +66,7 @@ | multipart_storage_class_test | 3 | ✅ | | namespace_lock_quorum_test | 2 | | | negative_sigv4_test | 6 | ✅ | -| notification_webhook_test | 2 | ✅ | +| notification_webhook_test | 3 | ✅ | | object_lambda_test | 16 | | | object_lock | 33 | | | overwrite_cleanup_regression_test | 1 | | @@ -88,6 +88,4 @@ | tls_hot_reload_test | 1 | ✅ | | version_id_regression_test | 10 | ✅ | -`notification_webhook_test` also has 1 ignored store-and-forward regression tracked by rustfs#4852; ignored tests are excluded from the active counts above. - -**Total listed: 527 tests across 70 modules · PR smoke subset: 147 tests / 33 modules** (31 full modules + 18 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 28 tests** · generated 2026-07-30. +**Total listed: 528 tests across 70 modules · PR smoke subset: 148 tests / 33 modules** (31 full modules + 18 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 28 tests** · generated 2026-08-04.