fix(audit): harden reload ordering, start race, paused drops, and batch observability (#4497)

Follow-up hardening for the audit control plane. The ABBA deadlock
(backlog#961), dispatch failure propagation (backlog#962), and credential
header redaction (backlog#963) already landed on main; this change addresses
the remaining audit-side findings.

- backlog#970: reload/commit now shuts down existing replay workers and closes
  old targets *before* activating the replacement set, so old and new workers
  never drain the same store concurrently and re-deliver entries. Extracted a
  state-neutral `shutdown_runtime_targets` helper (registry-then-cancellers
  lock order preserved).
- backlog#978: `start()` claims the `Starting` transition atomically under the
  state lock, closing the check-then-act race that could double-activate;
  `dispatch()` no longer returns Ok while paused, it surfaces an explicit
  `AuditError::Paused` so the audit trail is not silently corrupted.
- backlog#984 (audit part): `dispatch_audit_log` performs a single state read
  and interprets not-running/paused as a deliberate skip while surfacing real
  delivery failures; `dispatch_batch` records the same observability signals as
  single dispatch; documented the unordered cross-target fan-out.

Adds unit tests: paused dispatch returns Err, concurrent start does not hang or
double-activate, commit closes old targets before installing new, and
dispatch/dispatch_batch delivery-outcome coverage (all-fail -> Err, partial ->
Ok, all-success -> Ok).

Relates to rustfs/backlog#970
Relates to rustfs/backlog#978
Relates to rustfs/backlog#984

Co-authored-by: heihutu <heihutu@gmail.com>

crates/audit/src/global.rs

@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use crate::{AuditEntry, AuditResult, AuditSystem, system::AuditTargetMetricSnapshot};
+use crate::{AuditEntry, AuditError, AuditResult, AuditSystem, system::AuditTargetMetricSnapshot};
 use rustfs_config::server_config::Config;
 use std::sync::{Arc, OnceLock};
 use tracing::{debug, error, trace};
@@ -78,10 +78,27 @@ pub async fn resume_audit_system() -> AuditResult<()> {
 
 /// Dispatch an audit log entry to all targets
 pub async fn dispatch_audit_log(entry: Arc<AuditEntry>) -> AuditResult<()> {
-    if let Some(system) = audit_system() {
-        if system.is_running().await {
-            system.dispatch(entry).await
-        } else {
+    let Some(system) = audit_system() else {
+        debug!(
+            event = EVENT_AUDIT_ENTRY_DROPPED,
+            component = LOG_COMPONENT_AUDIT,
+            subsystem = LOG_SUBSYSTEM_GLOBAL,
+            reason = "system_not_initialized",
+            "Dropped audit entry"
+        );
+        return Ok(());
+    };
+
+    // Single state read (backlog#984): the previous code checked `is_running()`
+    // and then called `dispatch()`, which re-read the state. Between the two
+    // reads the system could transition (e.g. Running -> Stopping) and
+    // `dispatch()` would return an error the caller never expected. Let
+    // `dispatch()` be the single authority on the current state and interpret
+    // its "not accepting" errors as a deliberate skip, while still surfacing
+    // real delivery failures (backlog#962).
+    match system.dispatch(entry).await {
+        Ok(()) => Ok(()),
+        Err(AuditError::NotInitialized(_)) | Err(AuditError::Paused) => {
             trace!(
                 event = EVENT_AUDIT_ENTRY_DROPPED,
                 component = LOG_COMPONENT_AUDIT,
@@ -91,15 +108,7 @@ pub async fn dispatch_audit_log(entry: Arc<AuditEntry>) -> AuditResult<()> {
             );
             Ok(())
         }
-    } else {
-        debug!(
-            event = EVENT_AUDIT_ENTRY_DROPPED,
-            component = LOG_COMPONENT_AUDIT,
-            subsystem = LOG_SUBSYSTEM_GLOBAL,
-            reason = "system_not_initialized",
-            "Dropped audit entry"
-        );
-        Ok(())
+        Err(e) => Err(e),
     }
 }