RustFS rustfs-audit Complete Implementation with Enterprise Observability (#557)

* Initial plan * Implement core audit system with multi-target fan-out and configuration management Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * Changes before error encountered Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * Complete audit system with comprehensive observability and test coverage Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * improve code * fix * improve code * fix test * fix test * fix * add `rustfs-audit` to `rustfs` * upgrade crate version * fmt * fmt * fix --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com>
2026-08-13 00:26:53 +00:00 · 2025-09-24 08:23:46 +08:00
parent 08aeca89ef
commit 29b0935be7
45 changed files with 4139 additions and 313 deletions
@@ -141,9 +141,9 @@ impl TransitionClient {
        let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;

        //#[cfg(feature = "ring")]
-        //let _ = rustls::crypto::ring::default_provider().install_default();
+        let _ = rustls::crypto::ring::default_provider().install_default();
        //#[cfg(feature = "aws-lc-rs")]
-        let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
+        // let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();

        let scheme = endpoint_url.scheme();
        let client;
@@ -13,17 +13,18 @@
 //  limitations under the License.

 use crate::config::{KV, KVS};
-use rustfs_config::audit::{
-    WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT,
-    WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_SIZE, WEBHOOK_RETRY_INTERVAL,
+use rustfs_config::{
+    COMMENT_KEY, DEFAULT_DIR, DEFAULT_LIMIT, ENABLE_KEY, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
+    MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN,
+    WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY,
+    WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL,
 };
-use rustfs_config::{DEFAULT_DIR, DEFAULT_LIMIT, ENABLE_KEY, EnableState};
 use std::sync::LazyLock;

 #[allow(dead_code)]
 #[allow(clippy::declare_interior_mutable_const)]
 /// Default KVS for audit webhook settings.
-pub const DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
+pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
    KVS(vec![
        KV {
            key: ENABLE_KEY.to_owned(),
@@ -56,7 +57,7 @@ pub const DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
            hidden_if_empty: false,
        },
        KV {
-            key: WEBHOOK_QUEUE_SIZE.to_owned(),
+            key: WEBHOOK_QUEUE_LIMIT.to_owned(),
            value: DEFAULT_LIMIT.to_string(),
            hidden_if_empty: false,
        },
@@ -82,3 +83,66 @@ pub const DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
        },
    ])
 });
+
+#[allow(dead_code)]
+#[allow(clippy::declare_interior_mutable_const)]
+/// Default KVS for audit MQTT settings.
+pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: MQTT_BROKER.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: MQTT_TOPIC.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: MQTT_USERNAME.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: MQTT_PASSWORD.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true, // Sensitive field
+        },
+        KV {
+            key: MQTT_QOS.to_owned(),
+            value: "1".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: MQTT_KEEP_ALIVE_INTERVAL.to_owned(),
+            value: "60s".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: MQTT_RECONNECT_INTERVAL.to_owned(),
+            value: "5s".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: MQTT_QUEUE_DIR.to_owned(),
+            value: DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: MQTT_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});
@@ -24,6 +24,7 @@ use crate::store::ECStore;
 use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate};
 use rustfs_config::COMMENT_KEY;
 use rustfs_config::DEFAULT_DELIMITER;
+use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
 use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -222,8 +223,10 @@ pub fn init() {
    kvs.insert(STORAGE_CLASS_SUB_SYS.to_owned(), storageclass::DEFAULT_KVS.clone());
    // New: Loading default configurations for notify_webhook and notify_mqtt
    // Referring subsystem names through constants to improve the readability and maintainability of the code
-    kvs.insert(NOTIFY_WEBHOOK_SUB_SYS.to_owned(), notify::DEFAULT_WEBHOOK_KVS.clone());
-    kvs.insert(NOTIFY_MQTT_SUB_SYS.to_owned(), notify::DEFAULT_MQTT_KVS.clone());
+    kvs.insert(NOTIFY_WEBHOOK_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_WEBHOOK_KVS.clone());
+    kvs.insert(AUDIT_WEBHOOK_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_WEBHOOK_KVS.clone());
+    kvs.insert(NOTIFY_MQTT_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_MQTT_KVS.clone());
+    kvs.insert(AUDIT_MQTT_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_MQTT_KVS.clone());

    // Register all default configurations
    register_default_kvs(kvs)
@@ -13,17 +13,16 @@
 // limitations under the License.

 use crate::config::{KV, KVS};
-use rustfs_config::notify::{
-    MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL,
-    MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR,
-    WEBHOOK_QUEUE_LIMIT,
+use rustfs_config::{
+    COMMENT_KEY, DEFAULT_DIR, DEFAULT_LIMIT, ENABLE_KEY, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
+    MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN,
+    WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
 };
-use rustfs_config::{COMMENT_KEY, DEFAULT_DIR, DEFAULT_LIMIT, ENABLE_KEY, EnableState};
 use std::sync::LazyLock;

 /// The default configuration collection of webhooks，
 /// Initialized only once during the program life cycle, enabling high-performance lazy loading.
-pub static DEFAULT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
+pub static DEFAULT_NOTIFY_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
    KVS(vec![
        KV {
            key: ENABLE_KEY.to_owned(),
@@ -70,7 +69,7 @@ pub static DEFAULT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
 });

 /// MQTT's default configuration collection
-pub static DEFAULT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
+pub static DEFAULT_NOTIFY_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
    KVS(vec![
        KV {
            key: ENABLE_KEY.to_owned(),