rustfs/crates/e2e_test/src/notification_startup_regression_test.rs

// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Regression tests for the event notification startup race.
//!
//! Covers the recurring pattern where webhook/audit targets fail to load at boot
//! due to startup ordering (notification runtime starts before server config is
//! loaded). This has regressed 9+ times across beta.3 ~ beta.12.
//!
//! ## Regression Issues
//!
//! - rustfs#5387: webhook notifications broken again in beta.9+
//! - rustfs#5681: Audit webhook targets are not loaded at boot
//! - rustfs#5401: Event Destinations broken again
//! - rustfs#5183: Audit webhooks stay offline after restart
//! - rustfs#5115: init_event_notifier loses startup race against server config load
//! - rustfs#4796: Pulsar event destinations offline after restart
//! - rustfs#5428: MQTT bucket notifications stop on restarted cluster node

#[cfg(test)]
mod tests {
    use crate::common::{RustFSTestEnvironment, init_logging};
    use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
    use serial_test::serial;
    use std::error::Error;
    use tracing::info;

    type TestResult = Result<(), Box<dyn Error + Send + Sync>>;

    /// RT-01: Verify that the notification runtime initializes correctly at boot.
    ///
    /// Regression pattern: notification runtime initializes before server config
    /// is fully loaded, causing webhook targets to never come online.
    ///
    /// This test verifies the startup ordering by checking that the server
    /// starts successfully with notification enabled and can serve S3 requests.
    /// A full webhook delivery test is in notification_webhook_test.rs.
    #[tokio::test]
    #[serial]
    async fn test_notification_enabled_server_starts_cleanly() -> TestResult {
        init_logging();
        info!("RT-01: notification enabled server starts cleanly");

        let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
        env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false"), ("RUSTFS_NOTIFY_ENABLE", "true")])
            .await
            .expect("start RustFS with notifications enabled");

        let client = env.create_s3_client();
        let bucket = "rt01-notify-startup";

        // Server should be healthy and able to serve S3 requests
        client
            .create_bucket()
            .bucket(bucket)
            .send()
            .await
            .expect("create bucket with notifications enabled");

        client
            .put_object()
            .bucket(bucket)
            .key("test.txt")
            .body(aws_sdk_s3::primitives::ByteStream::from_static(b"test"))
            .send()
            .await
            .expect("put object with notifications enabled");

        info!("RT-01 PASS: notification enabled server starts and serves S3");
        Ok(())
    }

    /// RT-02: Verify notification config persists after server restart.
    ///
    /// Regression pattern: after a node restart, notification targets stay
    /// offline permanently because the config is not re-loaded.
    ///
    /// Steps:
    /// 1. Start server with notification enabled
    /// 2. Create bucket and configure notification
    /// 3. Restart server
    /// 4. Verify notification config still exists
    #[tokio::test]
    #[serial]
    async fn test_notification_config_survives_restart() -> TestResult {
        init_logging();
        info!("RT-02: notification config survives restart");

        let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
        env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false"), ("RUSTFS_NOTIFY_ENABLE", "true")])
            .await
            .expect("start RustFS");

        let client = env.create_s3_client();
        let bucket = "rt02-notify-restart";

        client.create_bucket().bucket(bucket).send().await.expect("create bucket");

        // Enable versioning (required for notification configuration)
        client
            .put_bucket_versioning()
            .bucket(bucket)
            .versioning_configuration(
                VersioningConfiguration::builder()
                    .status(BucketVersioningStatus::Enabled)
                    .build(),
            )
            .send()
            .await
            .expect("enable versioning");

        // Note: We can't fully test notification config persistence without a
        // configured target. But we verify the server restarts cleanly with
        // notification enabled, which is the core regression scenario.
        env.restart_server_preserving_data(vec![], &[])
            .await
            .expect("restart RustFS with notifications enabled");

        // Verify bucket still exists and is accessible after restart
        let list = client
            .list_objects_v2()
            .bucket(bucket)
            .send()
            .await
            .expect("list objects after restart");

        assert!(list.contents().is_empty(), "RT-02: bucket should be empty after restart");

        // Verify we can still write objects (notification runtime initialized)
        client
            .put_object()
            .bucket(bucket)
            .key("after-restart.txt")
            .body(aws_sdk_s3::primitives::ByteStream::from_static(b"post-restart"))
            .send()
            .await
            .expect("put object after restart — notification runtime must be initialized");

        info!("RT-02 PASS: server with notifications survives restart");
        Ok(())
    }
}