RustFS rustfs-audit Complete Implementation with Enterprise Observability (#557)

* Initial plan

* Implement core audit system with multi-target fan-out and configuration management

Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

* Changes before error encountered

Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

* Complete audit system with comprehensive observability and test coverage

Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

* improve code

* fix

* improve code

* fix test

* fix test

* fix

* add `rustfs-audit` to `rustfs`

* upgrade crate version

* fmt

* fmt

* fix

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>

crates/config/src/audit/mod.rs

@@ -13,19 +13,24 @@
 //  limitations under the License.
 
 //! Audit configuration module
-//! //! This module defines the configuration for audit systems, including
-//! webhook and other audit-related settings.
+//! This module defines the configuration for audit systems, including
+//! webhook and MQTT audit-related settings.
+
+pub(crate) mod mqtt;
+pub(crate) mod webhook;
+
+pub use mqtt::*;
+pub use webhook::*;
+
+use crate::DEFAULT_DELIMITER;
+// --- Audit subsystem identifiers ---
+pub const AUDIT_PREFIX: &str = "audit";
+
+pub const AUDIT_ROUTE_PREFIX: &str = const_str::concat!(AUDIT_PREFIX, DEFAULT_DELIMITER);
+
 pub const AUDIT_WEBHOOK_SUB_SYS: &str = "audit_webhook";
+pub const AUDIT_MQTT_SUB_SYS: &str = "mqtt_webhook";
 
 pub const AUDIT_STORE_EXTENSION: &str = ".audit";
-
-pub const WEBHOOK_ENDPOINT: &str = "endpoint";
-pub const WEBHOOK_AUTH_TOKEN: &str = "auth_token";
-pub const WEBHOOK_CLIENT_CERT: &str = "client_cert";
-pub const WEBHOOK_CLIENT_KEY: &str = "client_key";
-pub const WEBHOOK_BATCH_SIZE: &str = "batch_size";
-pub const WEBHOOK_QUEUE_SIZE: &str = "queue_size";
-pub const WEBHOOK_QUEUE_DIR: &str = "queue_dir";
-pub const WEBHOOK_MAX_RETRY: &str = "max_retry";
-pub const WEBHOOK_RETRY_INTERVAL: &str = "retry_interval";
-pub const WEBHOOK_HTTP_TIMEOUT: &str = "http_timeout";
+#[allow(dead_code)]
+pub const AUDIT_SUB_SYSTEMS: &[&str] = &[AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS];

crates/config/src/audit/mqtt.rs

@@ -0,0 +1,54 @@
+//  Copyright 2024 RustFS Team
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+// MQTT Environment Variables
+pub const ENV_AUDIT_MQTT_ENABLE: &str = "RUSTFS_AUDIT_MQTT_ENABLE";
+pub const ENV_AUDIT_MQTT_BROKER: &str = "RUSTFS_AUDIT_MQTT_BROKER";
+pub const ENV_AUDIT_MQTT_TOPIC: &str = "RUSTFS_AUDIT_MQTT_TOPIC";
+pub const ENV_AUDIT_MQTT_QOS: &str = "RUSTFS_AUDIT_MQTT_QOS";
+pub const ENV_AUDIT_MQTT_USERNAME: &str = "RUSTFS_AUDIT_MQTT_USERNAME";
+pub const ENV_AUDIT_MQTT_PASSWORD: &str = "RUSTFS_AUDIT_MQTT_PASSWORD";
+pub const ENV_AUDIT_MQTT_RECONNECT_INTERVAL: &str = "RUSTFS_AUDIT_MQTT_RECONNECT_INTERVAL";
+pub const ENV_AUDIT_MQTT_KEEP_ALIVE_INTERVAL: &str = "RUSTFS_AUDIT_MQTT_KEEP_ALIVE_INTERVAL";
+pub const ENV_AUDIT_MQTT_QUEUE_DIR: &str = "RUSTFS_AUDIT_MQTT_QUEUE_DIR";
+pub const ENV_AUDIT_MQTT_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_MQTT_QUEUE_LIMIT";
+
+/// A list of all valid configuration keys for an MQTT target.
+pub const ENV_AUDIT_MQTT_KEYS: &[&str; 10] = &[
+    ENV_AUDIT_MQTT_ENABLE,
+    ENV_AUDIT_MQTT_BROKER,
+    ENV_AUDIT_MQTT_TOPIC,
+    ENV_AUDIT_MQTT_QOS,
+    ENV_AUDIT_MQTT_USERNAME,
+    ENV_AUDIT_MQTT_PASSWORD,
+    ENV_AUDIT_MQTT_RECONNECT_INTERVAL,
+    ENV_AUDIT_MQTT_KEEP_ALIVE_INTERVAL,
+    ENV_AUDIT_MQTT_QUEUE_DIR,
+    ENV_AUDIT_MQTT_QUEUE_LIMIT,
+];
+
+/// A list of all valid configuration keys for an MQTT target.
+pub const AUDIT_MQTT_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::MQTT_BROKER,
+    crate::MQTT_TOPIC,
+    crate::MQTT_QOS,
+    crate::MQTT_USERNAME,
+    crate::MQTT_PASSWORD,
+    crate::MQTT_RECONNECT_INTERVAL,
+    crate::MQTT_KEEP_ALIVE_INTERVAL,
+    crate::MQTT_QUEUE_DIR,
+    crate::MQTT_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];

crates/config/src/audit/webhook.rs

@@ -0,0 +1,45 @@
+//  Copyright 2024 RustFS Team
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+// Webhook Environment Variables
+pub const ENV_AUDIT_WEBHOOK_ENABLE: &str = "RUSTFS_AUDIT_WEBHOOK_ENABLE";
+pub const ENV_AUDIT_WEBHOOK_ENDPOINT: &str = "RUSTFS_AUDIT_WEBHOOK_ENDPOINT";
+pub const ENV_AUDIT_WEBHOOK_AUTH_TOKEN: &str = "RUSTFS_AUDIT_WEBHOOK_AUTH_TOKEN";
+pub const ENV_AUDIT_WEBHOOK_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_WEBHOOK_QUEUE_LIMIT";
+pub const ENV_AUDIT_WEBHOOK_QUEUE_DIR: &str = "RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR";
+pub const ENV_AUDIT_WEBHOOK_CLIENT_CERT: &str = "RUSTFS_AUDIT_WEBHOOK_CLIENT_CERT";
+pub const ENV_AUDIT_WEBHOOK_CLIENT_KEY: &str = "RUSTFS_AUDIT_WEBHOOK_CLIENT_KEY";
+
+/// List of all environment variable keys for a webhook target.
+pub const ENV_AUDIT_WEBHOOK_KEYS: &[&str; 7] = &[
+    ENV_AUDIT_WEBHOOK_ENABLE,
+    ENV_AUDIT_WEBHOOK_ENDPOINT,
+    ENV_AUDIT_WEBHOOK_AUTH_TOKEN,
+    ENV_AUDIT_WEBHOOK_QUEUE_LIMIT,
+    ENV_AUDIT_WEBHOOK_QUEUE_DIR,
+    ENV_AUDIT_WEBHOOK_CLIENT_CERT,
+    ENV_AUDIT_WEBHOOK_CLIENT_KEY,
+];
+
+/// A list of all valid configuration keys for a webhook target.
+pub const AUDIT_WEBHOOK_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::WEBHOOK_ENDPOINT,
+    crate::WEBHOOK_AUTH_TOKEN,
+    crate::WEBHOOK_QUEUE_LIMIT,
+    crate::WEBHOOK_QUEUE_DIR,
+    crate::WEBHOOK_CLIENT_CERT,
+    crate::WEBHOOK_CLIENT_KEY,
+    crate::COMMENT_KEY,
+];

crates/config/src/constants/mod.rs

@@ -15,4 +15,5 @@
 pub(crate) mod app;
 pub(crate) mod console;
 pub(crate) mod env;
+pub(crate) mod targets;
 pub(crate) mod tls;

crates/config/src/constants/targets.rs

@@ -0,0 +1,34 @@
+//  Copyright 2024 RustFS Team
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+pub const WEBHOOK_ENDPOINT: &str = "endpoint";
+pub const WEBHOOK_AUTH_TOKEN: &str = "auth_token";
+pub const WEBHOOK_CLIENT_CERT: &str = "client_cert";
+pub const WEBHOOK_CLIENT_KEY: &str = "client_key";
+pub const WEBHOOK_BATCH_SIZE: &str = "batch_size";
+pub const WEBHOOK_QUEUE_LIMIT: &str = "queue_limit";
+pub const WEBHOOK_QUEUE_DIR: &str = "queue_dir";
+pub const WEBHOOK_MAX_RETRY: &str = "max_retry";
+pub const WEBHOOK_RETRY_INTERVAL: &str = "retry_interval";
+pub const WEBHOOK_HTTP_TIMEOUT: &str = "http_timeout";
+
+pub const MQTT_BROKER: &str = "broker";
+pub const MQTT_TOPIC: &str = "topic";
+pub const MQTT_QOS: &str = "qos";
+pub const MQTT_USERNAME: &str = "username";
+pub const MQTT_PASSWORD: &str = "password";
+pub const MQTT_RECONNECT_INTERVAL: &str = "reconnect_interval";
+pub const MQTT_KEEP_ALIVE_INTERVAL: &str = "keep_alive_interval";
+pub const MQTT_QUEUE_DIR: &str = "queue_dir";
+pub const MQTT_QUEUE_LIMIT: &str = "queue_limit";

crates/config/src/lib.rs

@@ -21,6 +21,8 @@ pub use constants::console::*;
 #[cfg(feature = "constants")]
 pub use constants::env::*;
 #[cfg(feature = "constants")]
+pub use constants::targets::*;
+#[cfg(feature = "constants")]
 pub use constants::tls::*;
 #[cfg(feature = "audit")]
 pub mod audit;

crates/config/src/notify/mod.rs

@@ -22,12 +22,14 @@ pub use mqtt::*;
 pub use store::*;
 pub use webhook::*;
 
+use crate::DEFAULT_DELIMITER;
+
 // --- Configuration Constants ---
 pub const DEFAULT_TARGET: &str = "1";
 
 pub const NOTIFY_PREFIX: &str = "notify";
 
-pub const NOTIFY_ROUTE_PREFIX: &str = const_str::concat!(NOTIFY_PREFIX, "_");
+pub const NOTIFY_ROUTE_PREFIX: &str = const_str::concat!(NOTIFY_PREFIX, DEFAULT_DELIMITER);
 
 #[allow(dead_code)]
 pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS];

crates/config/src/notify/mqtt.rs

@@ -12,55 +12,42 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::{COMMENT_KEY, ENABLE_KEY};
-
-// MQTT Keys
-pub const MQTT_BROKER: &str = "broker";
-pub const MQTT_TOPIC: &str = "topic";
-pub const MQTT_QOS: &str = "qos";
-pub const MQTT_USERNAME: &str = "username";
-pub const MQTT_PASSWORD: &str = "password";
-pub const MQTT_RECONNECT_INTERVAL: &str = "reconnect_interval";
-pub const MQTT_KEEP_ALIVE_INTERVAL: &str = "keep_alive_interval";
-pub const MQTT_QUEUE_DIR: &str = "queue_dir";
-pub const MQTT_QUEUE_LIMIT: &str = "queue_limit";
-
 /// A list of all valid configuration keys for an MQTT target.
 pub const NOTIFY_MQTT_KEYS: &[&str] = &[
-    ENABLE_KEY,
-    MQTT_BROKER,
-    MQTT_TOPIC,
-    MQTT_QOS,
-    MQTT_USERNAME,
-    MQTT_PASSWORD,
-    MQTT_RECONNECT_INTERVAL,
-    MQTT_KEEP_ALIVE_INTERVAL,
-    MQTT_QUEUE_DIR,
-    MQTT_QUEUE_LIMIT,
-    COMMENT_KEY,
+    crate::ENABLE_KEY,
+    crate::MQTT_BROKER,
+    crate::MQTT_TOPIC,
+    crate::MQTT_QOS,
+    crate::MQTT_USERNAME,
+    crate::MQTT_PASSWORD,
+    crate::MQTT_RECONNECT_INTERVAL,
+    crate::MQTT_KEEP_ALIVE_INTERVAL,
+    crate::MQTT_QUEUE_DIR,
+    crate::MQTT_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
 ];
 
 // MQTT Environment Variables
-pub const ENV_MQTT_ENABLE: &str = "RUSTFS_NOTIFY_MQTT_ENABLE";
-pub const ENV_MQTT_BROKER: &str = "RUSTFS_NOTIFY_MQTT_BROKER";
-pub const ENV_MQTT_TOPIC: &str = "RUSTFS_NOTIFY_MQTT_TOPIC";
-pub const ENV_MQTT_QOS: &str = "RUSTFS_NOTIFY_MQTT_QOS";
-pub const ENV_MQTT_USERNAME: &str = "RUSTFS_NOTIFY_MQTT_USERNAME";
-pub const ENV_MQTT_PASSWORD: &str = "RUSTFS_NOTIFY_MQTT_PASSWORD";
-pub const ENV_MQTT_RECONNECT_INTERVAL: &str = "RUSTFS_NOTIFY_MQTT_RECONNECT_INTERVAL";
-pub const ENV_MQTT_KEEP_ALIVE_INTERVAL: &str = "RUSTFS_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL";
-pub const ENV_MQTT_QUEUE_DIR: &str = "RUSTFS_NOTIFY_MQTT_QUEUE_DIR";
-pub const ENV_MQTT_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_MQTT_QUEUE_LIMIT";
+pub const ENV_NOTIFY_MQTT_ENABLE: &str = "RUSTFS_NOTIFY_MQTT_ENABLE";
+pub const ENV_NOTIFY_MQTT_BROKER: &str = "RUSTFS_NOTIFY_MQTT_BROKER";
+pub const ENV_NOTIFY_MQTT_TOPIC: &str = "RUSTFS_NOTIFY_MQTT_TOPIC";
+pub const ENV_NOTIFY_MQTT_QOS: &str = "RUSTFS_NOTIFY_MQTT_QOS";
+pub const ENV_NOTIFY_MQTT_USERNAME: &str = "RUSTFS_NOTIFY_MQTT_USERNAME";
+pub const ENV_NOTIFY_MQTT_PASSWORD: &str = "RUSTFS_NOTIFY_MQTT_PASSWORD";
+pub const ENV_NOTIFY_MQTT_RECONNECT_INTERVAL: &str = "RUSTFS_NOTIFY_MQTT_RECONNECT_INTERVAL";
+pub const ENV_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL: &str = "RUSTFS_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL";
+pub const ENV_NOTIFY_MQTT_QUEUE_DIR: &str = "RUSTFS_NOTIFY_MQTT_QUEUE_DIR";
+pub const ENV_NOTIFY_MQTT_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_MQTT_QUEUE_LIMIT";
 
 pub const ENV_NOTIFY_MQTT_KEYS: &[&str; 10] = &[
-    ENV_MQTT_ENABLE,
-    ENV_MQTT_BROKER,
-    ENV_MQTT_TOPIC,
-    ENV_MQTT_QOS,
-    ENV_MQTT_USERNAME,
-    ENV_MQTT_PASSWORD,
-    ENV_MQTT_RECONNECT_INTERVAL,
-    ENV_MQTT_KEEP_ALIVE_INTERVAL,
-    ENV_MQTT_QUEUE_DIR,
-    ENV_MQTT_QUEUE_LIMIT,
+    ENV_NOTIFY_MQTT_ENABLE,
+    ENV_NOTIFY_MQTT_BROKER,
+    ENV_NOTIFY_MQTT_TOPIC,
+    ENV_NOTIFY_MQTT_QOS,
+    ENV_NOTIFY_MQTT_USERNAME,
+    ENV_NOTIFY_MQTT_PASSWORD,
+    ENV_NOTIFY_MQTT_RECONNECT_INTERVAL,
+    ENV_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL,
+    ENV_NOTIFY_MQTT_QUEUE_DIR,
+    ENV_NOTIFY_MQTT_QUEUE_LIMIT,
 ];

crates/config/src/notify/webhook.rs

@@ -12,43 +12,33 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::{COMMENT_KEY, ENABLE_KEY};
-
-// Webhook Keys
-pub const WEBHOOK_ENDPOINT: &str = "endpoint";
-pub const WEBHOOK_AUTH_TOKEN: &str = "auth_token";
-pub const WEBHOOK_QUEUE_LIMIT: &str = "queue_limit";
-pub const WEBHOOK_QUEUE_DIR: &str = "queue_dir";
-pub const WEBHOOK_CLIENT_CERT: &str = "client_cert";
-pub const WEBHOOK_CLIENT_KEY: &str = "client_key";
-
 /// A list of all valid configuration keys for a webhook target.
 pub const NOTIFY_WEBHOOK_KEYS: &[&str] = &[
-    ENABLE_KEY,
-    WEBHOOK_ENDPOINT,
-    WEBHOOK_AUTH_TOKEN,
-    WEBHOOK_QUEUE_LIMIT,
-    WEBHOOK_QUEUE_DIR,
-    WEBHOOK_CLIENT_CERT,
-    WEBHOOK_CLIENT_KEY,
-    COMMENT_KEY,
+    crate::ENABLE_KEY,
+    crate::WEBHOOK_ENDPOINT,
+    crate::WEBHOOK_AUTH_TOKEN,
+    crate::WEBHOOK_QUEUE_LIMIT,
+    crate::WEBHOOK_QUEUE_DIR,
+    crate::WEBHOOK_CLIENT_CERT,
+    crate::WEBHOOK_CLIENT_KEY,
+    crate::COMMENT_KEY,
 ];
 
 // Webhook Environment Variables
-pub const ENV_WEBHOOK_ENABLE: &str = "RUSTFS_NOTIFY_WEBHOOK_ENABLE";
-pub const ENV_WEBHOOK_ENDPOINT: &str = "RUSTFS_NOTIFY_WEBHOOK_ENDPOINT";
-pub const ENV_WEBHOOK_AUTH_TOKEN: &str = "RUSTFS_NOTIFY_WEBHOOK_AUTH_TOKEN";
-pub const ENV_WEBHOOK_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT";
-pub const ENV_WEBHOOK_QUEUE_DIR: &str = "RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR";
-pub const ENV_WEBHOOK_CLIENT_CERT: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_CERT";
-pub const ENV_WEBHOOK_CLIENT_KEY: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_KEY";
+pub const ENV_NOTIFY_WEBHOOK_ENABLE: &str = "RUSTFS_NOTIFY_WEBHOOK_ENABLE";
+pub const ENV_NOTIFY_WEBHOOK_ENDPOINT: &str = "RUSTFS_NOTIFY_WEBHOOK_ENDPOINT";
+pub const ENV_NOTIFY_WEBHOOK_AUTH_TOKEN: &str = "RUSTFS_NOTIFY_WEBHOOK_AUTH_TOKEN";
+pub const ENV_NOTIFY_WEBHOOK_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT";
+pub const ENV_NOTIFY_WEBHOOK_QUEUE_DIR: &str = "RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR";
+pub const ENV_NOTIFY_WEBHOOK_CLIENT_CERT: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_CERT";
+pub const ENV_NOTIFY_WEBHOOK_CLIENT_KEY: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_KEY";
 
 pub const ENV_NOTIFY_WEBHOOK_KEYS: &[&str; 7] = &[
-    ENV_WEBHOOK_ENABLE,
-    ENV_WEBHOOK_ENDPOINT,
-    ENV_WEBHOOK_AUTH_TOKEN,
-    ENV_WEBHOOK_QUEUE_LIMIT,
-    ENV_WEBHOOK_QUEUE_DIR,
-    ENV_WEBHOOK_CLIENT_CERT,
-    ENV_WEBHOOK_CLIENT_KEY,
+    ENV_NOTIFY_WEBHOOK_ENABLE,
+    ENV_NOTIFY_WEBHOOK_ENDPOINT,
+    ENV_NOTIFY_WEBHOOK_AUTH_TOKEN,
+    ENV_NOTIFY_WEBHOOK_QUEUE_LIMIT,
+    ENV_NOTIFY_WEBHOOK_QUEUE_DIR,
+    ENV_NOTIFY_WEBHOOK_CLIENT_CERT,
+    ENV_NOTIFY_WEBHOOK_CLIENT_KEY,
 ];