Files
rustfs/crates/audit/src/global.rs
T
houseme e44bece00d fix(audit): harden reload ordering, start race, paused drops, and batch observability (#4497)
Follow-up hardening for the audit control plane. The ABBA deadlock
(backlog#961), dispatch failure propagation (backlog#962), and credential
header redaction (backlog#963) already landed on main; this change addresses
the remaining audit-side findings.

- backlog#970: reload/commit now shuts down existing replay workers and closes
  old targets *before* activating the replacement set, so old and new workers
  never drain the same store concurrently and re-deliver entries. Extracted a
  state-neutral `shutdown_runtime_targets` helper (registry-then-cancellers
  lock order preserved).
- backlog#978: `start()` claims the `Starting` transition atomically under the
  state lock, closing the check-then-act race that could double-activate;
  `dispatch()` no longer returns Ok while paused, it surfaces an explicit
  `AuditError::Paused` so the audit trail is not silently corrupted.
- backlog#984 (audit part): `dispatch_audit_log` performs a single state read
  and interprets not-running/paused as a deliberate skip while surfacing real
  delivery failures; `dispatch_batch` records the same observability signals as
  single dispatch; documented the unordered cross-target fan-out.

Adds unit tests: paused dispatch returns Err, concurrent start does not hang or
double-activate, commit closes old targets before installing new, and
dispatch/dispatch_batch delivery-outcome coverage (all-fail -> Err, partial ->
Ok, all-success -> Ok).

Relates to rustfs/backlog#970
Relates to rustfs/backlog#978
Relates to rustfs/backlog#984

Co-authored-by: heihutu <heihutu@gmail.com>
2026-07-08 17:05:21 +00:00

166 lines
5.7 KiB
Rust

// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{AuditEntry, AuditError, AuditResult, AuditSystem, system::AuditTargetMetricSnapshot};
use rustfs_config::server_config::Config;
use std::sync::{Arc, OnceLock};
use tracing::{debug, error, trace};
const LOG_COMPONENT_AUDIT: &str = "audit";
const LOG_SUBSYSTEM_GLOBAL: &str = "global";
const EVENT_AUDIT_GLOBAL_SKIPPED: &str = "audit_global_skipped";
const EVENT_AUDIT_ENTRY_DROPPED: &str = "audit_entry_dropped";
const EVENT_AUDIT_DISPATCH_FAILED: &str = "audit_dispatch_failed";
/// Global audit system instance
static AUDIT_SYSTEM: OnceLock<Arc<AuditSystem>> = OnceLock::new();
/// Initialize the global audit system
pub fn init_audit_system() -> Arc<AuditSystem> {
AUDIT_SYSTEM.get_or_init(|| Arc::new(AuditSystem::new())).clone()
}
/// Get the global audit system instance
pub fn audit_system() -> Option<Arc<AuditSystem>> {
AUDIT_SYSTEM.get().cloned()
}
/// A helper macro for executing closures if the global audit system is initialized.
/// If not initialized, log a warning and return `Ok(())`.
macro_rules! with_audit_system {
($async_closure:expr) => {
if let Some(system) = audit_system() {
(async move { $async_closure(system).await }).await
} else {
debug!(
event = EVENT_AUDIT_GLOBAL_SKIPPED,
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_GLOBAL,
reason = "system_not_initialized",
"Skipped audit system operation"
);
Ok(())
}
};
}
/// Start the global audit system with configuration
pub async fn start_audit_system(config: Config) -> AuditResult<()> {
let system = init_audit_system();
system.start(config).await
}
/// Stop the global audit system
pub async fn stop_audit_system() -> AuditResult<()> {
with_audit_system!(|system: Arc<AuditSystem>| async move { system.close().await })
}
/// Pause the global audit system
pub async fn pause_audit_system() -> AuditResult<()> {
with_audit_system!(|system: Arc<AuditSystem>| async move { system.pause().await })
}
/// Resume the global audit system
pub async fn resume_audit_system() -> AuditResult<()> {
with_audit_system!(|system: Arc<AuditSystem>| async move { system.resume().await })
}
/// Dispatch an audit log entry to all targets
pub async fn dispatch_audit_log(entry: Arc<AuditEntry>) -> AuditResult<()> {
let Some(system) = audit_system() else {
debug!(
event = EVENT_AUDIT_ENTRY_DROPPED,
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_GLOBAL,
reason = "system_not_initialized",
"Dropped audit entry"
);
return Ok(());
};
// Single state read (backlog#984): the previous code checked `is_running()`
// and then called `dispatch()`, which re-read the state. Between the two
// reads the system could transition (e.g. Running -> Stopping) and
// `dispatch()` would return an error the caller never expected. Let
// `dispatch()` be the single authority on the current state and interpret
// its "not accepting" errors as a deliberate skip, while still surfacing
// real delivery failures (backlog#962).
match system.dispatch(entry).await {
Ok(()) => Ok(()),
Err(AuditError::NotInitialized(_)) | Err(AuditError::Paused) => {
trace!(
event = EVENT_AUDIT_ENTRY_DROPPED,
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_GLOBAL,
reason = "system_not_running",
"Dropped audit entry"
);
Ok(())
}
Err(e) => Err(e),
}
}
/// Reload the global audit system configuration
pub async fn reload_audit_config(config: Config) -> AuditResult<()> {
with_audit_system!(|system: Arc<AuditSystem>| async move { system.reload_config(config).await })
}
/// Returns per-target audit delivery metrics for Prometheus collection.
pub async fn audit_target_metrics() -> Vec<AuditTargetMetricSnapshot> {
if let Some(system) = audit_system() {
system.snapshot_target_metrics().await
} else {
Vec::new()
}
}
/// Check if the global audit system is running
pub async fn is_audit_system_running() -> bool {
if let Some(system) = audit_system() {
system.is_running().await
} else {
false
}
}
/// AuditLogger singleton for easy access
pub struct AuditLogger;
impl AuditLogger {
/// Log an audit entry
pub async fn log(entry: AuditEntry) {
if let Err(e) = dispatch_audit_log(Arc::new(entry)).await {
error!(
event = EVENT_AUDIT_DISPATCH_FAILED,
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_GLOBAL,
error = %e,
"Failed to dispatch audit entry"
);
}
}
/// Check if audit logging is enabled
pub async fn is_enabled() -> bool {
is_audit_system_running().await
}
/// Get singleton instance
pub fn instance() -> &'static Self {
static INSTANCE: AuditLogger = AuditLogger;
&INSTANCE
}
}