// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //! Regression tests for the event notification startup race. //! //! Covers the recurring pattern where webhook/audit targets fail to load at boot //! due to startup ordering (notification runtime starts before server config is //! loaded). This has regressed 9+ times across beta.3 ~ beta.12. //! //! ## Regression Issues //! //! - rustfs#5387: webhook notifications broken again in beta.9+ //! - rustfs#5681: Audit webhook targets are not loaded at boot //! - rustfs#5401: Event Destinations broken again //! - rustfs#5183: Audit webhooks stay offline after restart //! - rustfs#5115: init_event_notifier loses startup race against server config load //! - rustfs#4796: Pulsar event destinations offline after restart //! - rustfs#5428: MQTT bucket notifications stop on restarted cluster node #[cfg(test)] mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; use serial_test::serial; use std::error::Error; use tracing::info; type TestResult = Result<(), Box>; /// RT-01: Verify that the notification runtime initializes correctly at boot. /// /// Regression pattern: notification runtime initializes before server config /// is fully loaded, causing webhook targets to never come online. /// /// This test verifies the startup ordering by checking that the server /// starts successfully with notification enabled and can serve S3 requests. /// A full webhook delivery test is in notification_webhook_test.rs. #[tokio::test] #[serial] async fn test_notification_enabled_server_starts_cleanly() -> TestResult { init_logging(); info!("RT-01: notification enabled server starts cleanly"); let mut env = RustFSTestEnvironment::new().await.expect("create test environment"); env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false"), ("RUSTFS_NOTIFY_ENABLE", "true")]) .await .expect("start RustFS with notifications enabled"); let client = env.create_s3_client(); let bucket = "rt01-notify-startup"; // Server should be healthy and able to serve S3 requests client .create_bucket() .bucket(bucket) .send() .await .expect("create bucket with notifications enabled"); client .put_object() .bucket(bucket) .key("test.txt") .body(aws_sdk_s3::primitives::ByteStream::from_static(b"test")) .send() .await .expect("put object with notifications enabled"); info!("RT-01 PASS: notification enabled server starts and serves S3"); Ok(()) } /// RT-02: Verify notification config persists after server restart. /// /// Regression pattern: after a node restart, notification targets stay /// offline permanently because the config is not re-loaded. /// /// Steps: /// 1. Start server with notification enabled /// 2. Create bucket and configure notification /// 3. Restart server /// 4. Verify notification config still exists #[tokio::test] #[serial] async fn test_notification_config_survives_restart() -> TestResult { init_logging(); info!("RT-02: notification config survives restart"); let mut env = RustFSTestEnvironment::new().await.expect("create test environment"); env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false"), ("RUSTFS_NOTIFY_ENABLE", "true")]) .await .expect("start RustFS"); let client = env.create_s3_client(); let bucket = "rt02-notify-restart"; client.create_bucket().bucket(bucket).send().await.expect("create bucket"); // Enable versioning (required for notification configuration) client .put_bucket_versioning() .bucket(bucket) .versioning_configuration( VersioningConfiguration::builder() .status(BucketVersioningStatus::Enabled) .build(), ) .send() .await .expect("enable versioning"); // Note: We can't fully test notification config persistence without a // configured target. But we verify the server restarts cleanly with // notification enabled, which is the core regression scenario. env.restart_server_preserving_data(vec![], &[]) .await .expect("restart RustFS with notifications enabled"); // Verify bucket still exists and is accessible after restart let list = client .list_objects_v2() .bucket(bucket) .send() .await .expect("list objects after restart"); assert!(list.contents().is_empty(), "RT-02: bucket should be empty after restart"); // Verify we can still write objects (notification runtime initialized) client .put_object() .bucket(bucket) .key("after-restart.txt") .body(aws_sdk_s3::primitives::ByteStream::from_static(b"post-restart")) .send() .await .expect("put object after restart — notification runtime must be initialized"); info!("RT-02 PASS: server with notifications survives restart"); Ok(()) } }