mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-02 19:39:17 +00:00
e44bece00d
Follow-up hardening for the audit control plane. The ABBA deadlock (backlog#961), dispatch failure propagation (backlog#962), and credential header redaction (backlog#963) already landed on main; this change addresses the remaining audit-side findings. - backlog#970: reload/commit now shuts down existing replay workers and closes old targets *before* activating the replacement set, so old and new workers never drain the same store concurrently and re-deliver entries. Extracted a state-neutral `shutdown_runtime_targets` helper (registry-then-cancellers lock order preserved). - backlog#978: `start()` claims the `Starting` transition atomically under the state lock, closing the check-then-act race that could double-activate; `dispatch()` no longer returns Ok while paused, it surfaces an explicit `AuditError::Paused` so the audit trail is not silently corrupted. - backlog#984 (audit part): `dispatch_audit_log` performs a single state read and interprets not-running/paused as a deliberate skip while surfacing real delivery failures; `dispatch_batch` records the same observability signals as single dispatch; documented the unordered cross-target fan-out. Adds unit tests: paused dispatch returns Err, concurrent start does not hang or double-activate, commit closes old targets before installing new, and dispatch/dispatch_batch delivery-outcome coverage (all-fail -> Err, partial -> Ok, all-success -> Ok). Relates to rustfs/backlog#970 Relates to rustfs/backlog#978 Relates to rustfs/backlog#984 Co-authored-by: heihutu <heihutu@gmail.com>
166 lines
5.7 KiB
Rust
166 lines
5.7 KiB
Rust
// Copyright 2024 RustFS Team
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
use crate::{AuditEntry, AuditError, AuditResult, AuditSystem, system::AuditTargetMetricSnapshot};
|
|
use rustfs_config::server_config::Config;
|
|
use std::sync::{Arc, OnceLock};
|
|
use tracing::{debug, error, trace};
|
|
|
|
const LOG_COMPONENT_AUDIT: &str = "audit";
|
|
const LOG_SUBSYSTEM_GLOBAL: &str = "global";
|
|
const EVENT_AUDIT_GLOBAL_SKIPPED: &str = "audit_global_skipped";
|
|
const EVENT_AUDIT_ENTRY_DROPPED: &str = "audit_entry_dropped";
|
|
const EVENT_AUDIT_DISPATCH_FAILED: &str = "audit_dispatch_failed";
|
|
|
|
/// Global audit system instance
|
|
static AUDIT_SYSTEM: OnceLock<Arc<AuditSystem>> = OnceLock::new();
|
|
|
|
/// Initialize the global audit system
|
|
pub fn init_audit_system() -> Arc<AuditSystem> {
|
|
AUDIT_SYSTEM.get_or_init(|| Arc::new(AuditSystem::new())).clone()
|
|
}
|
|
|
|
/// Get the global audit system instance
|
|
pub fn audit_system() -> Option<Arc<AuditSystem>> {
|
|
AUDIT_SYSTEM.get().cloned()
|
|
}
|
|
|
|
/// A helper macro for executing closures if the global audit system is initialized.
|
|
/// If not initialized, log a warning and return `Ok(())`.
|
|
macro_rules! with_audit_system {
|
|
($async_closure:expr) => {
|
|
if let Some(system) = audit_system() {
|
|
(async move { $async_closure(system).await }).await
|
|
} else {
|
|
debug!(
|
|
event = EVENT_AUDIT_GLOBAL_SKIPPED,
|
|
component = LOG_COMPONENT_AUDIT,
|
|
subsystem = LOG_SUBSYSTEM_GLOBAL,
|
|
reason = "system_not_initialized",
|
|
"Skipped audit system operation"
|
|
);
|
|
Ok(())
|
|
}
|
|
};
|
|
}
|
|
|
|
/// Start the global audit system with configuration
|
|
pub async fn start_audit_system(config: Config) -> AuditResult<()> {
|
|
let system = init_audit_system();
|
|
system.start(config).await
|
|
}
|
|
|
|
/// Stop the global audit system
|
|
pub async fn stop_audit_system() -> AuditResult<()> {
|
|
with_audit_system!(|system: Arc<AuditSystem>| async move { system.close().await })
|
|
}
|
|
|
|
/// Pause the global audit system
|
|
pub async fn pause_audit_system() -> AuditResult<()> {
|
|
with_audit_system!(|system: Arc<AuditSystem>| async move { system.pause().await })
|
|
}
|
|
|
|
/// Resume the global audit system
|
|
pub async fn resume_audit_system() -> AuditResult<()> {
|
|
with_audit_system!(|system: Arc<AuditSystem>| async move { system.resume().await })
|
|
}
|
|
|
|
/// Dispatch an audit log entry to all targets
|
|
pub async fn dispatch_audit_log(entry: Arc<AuditEntry>) -> AuditResult<()> {
|
|
let Some(system) = audit_system() else {
|
|
debug!(
|
|
event = EVENT_AUDIT_ENTRY_DROPPED,
|
|
component = LOG_COMPONENT_AUDIT,
|
|
subsystem = LOG_SUBSYSTEM_GLOBAL,
|
|
reason = "system_not_initialized",
|
|
"Dropped audit entry"
|
|
);
|
|
return Ok(());
|
|
};
|
|
|
|
// Single state read (backlog#984): the previous code checked `is_running()`
|
|
// and then called `dispatch()`, which re-read the state. Between the two
|
|
// reads the system could transition (e.g. Running -> Stopping) and
|
|
// `dispatch()` would return an error the caller never expected. Let
|
|
// `dispatch()` be the single authority on the current state and interpret
|
|
// its "not accepting" errors as a deliberate skip, while still surfacing
|
|
// real delivery failures (backlog#962).
|
|
match system.dispatch(entry).await {
|
|
Ok(()) => Ok(()),
|
|
Err(AuditError::NotInitialized(_)) | Err(AuditError::Paused) => {
|
|
trace!(
|
|
event = EVENT_AUDIT_ENTRY_DROPPED,
|
|
component = LOG_COMPONENT_AUDIT,
|
|
subsystem = LOG_SUBSYSTEM_GLOBAL,
|
|
reason = "system_not_running",
|
|
"Dropped audit entry"
|
|
);
|
|
Ok(())
|
|
}
|
|
Err(e) => Err(e),
|
|
}
|
|
}
|
|
|
|
/// Reload the global audit system configuration
|
|
pub async fn reload_audit_config(config: Config) -> AuditResult<()> {
|
|
with_audit_system!(|system: Arc<AuditSystem>| async move { system.reload_config(config).await })
|
|
}
|
|
|
|
/// Returns per-target audit delivery metrics for Prometheus collection.
|
|
pub async fn audit_target_metrics() -> Vec<AuditTargetMetricSnapshot> {
|
|
if let Some(system) = audit_system() {
|
|
system.snapshot_target_metrics().await
|
|
} else {
|
|
Vec::new()
|
|
}
|
|
}
|
|
|
|
/// Check if the global audit system is running
|
|
pub async fn is_audit_system_running() -> bool {
|
|
if let Some(system) = audit_system() {
|
|
system.is_running().await
|
|
} else {
|
|
false
|
|
}
|
|
}
|
|
|
|
/// AuditLogger singleton for easy access
|
|
pub struct AuditLogger;
|
|
|
|
impl AuditLogger {
|
|
/// Log an audit entry
|
|
pub async fn log(entry: AuditEntry) {
|
|
if let Err(e) = dispatch_audit_log(Arc::new(entry)).await {
|
|
error!(
|
|
event = EVENT_AUDIT_DISPATCH_FAILED,
|
|
component = LOG_COMPONENT_AUDIT,
|
|
subsystem = LOG_SUBSYSTEM_GLOBAL,
|
|
error = %e,
|
|
"Failed to dispatch audit entry"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// Check if audit logging is enabled
|
|
pub async fn is_enabled() -> bool {
|
|
is_audit_system_running().await
|
|
}
|
|
|
|
/// Get singleton instance
|
|
pub fn instance() -> &'static Self {
|
|
static INSTANCE: AuditLogger = AuditLogger;
|
|
&INSTANCE
|
|
}
|
|
}
|