fix(config): fence persisted config updates and reloads (#5512)

* fix(config): fence persisted config updates and reloads

* fix(ci): unblock config and e2e checks

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>

rustfs/src/admin/handlers/audit_runtime_config.rs

@@ -15,13 +15,15 @@
 use crate::admin::handlers::supervise_admin_mutation;
 use crate::admin::handlers::target_descriptor::AdminTargetSpec;
 use crate::admin::runtime_sources::{AppContext, current_app_context, current_object_store_handle_for_context};
+use crate::admin::service::config::with_runtime_config_reload_lock;
 use crate::admin::storage_api::config::{
-    read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_server_config_snapshot,
+    read_admin_config_without_migrate, read_admin_server_config_snapshot, read_existing_admin_server_config_no_lock,
+    save_admin_server_config_snapshot, with_admin_server_config_read_lock,
 };
 use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
 use rustfs_config::DEFAULT_DELIMITER;
 use rustfs_config::server_config::Config;
-use s3s::{S3Result, s3_error};
+use s3s::{S3Error, S3Result, s3_error};
 use tracing::warn;
 
 pub(crate) async fn load_server_config_from_store_for_context(context: Option<&AppContext>) -> S3Result<Config> {
@@ -48,6 +50,14 @@ fn has_any_audit_targets(specs: &[AdminTargetSpec], config: &Config) -> bool {
     })
 }
 
+fn audit_config_convergence_error(persisted: bool, error: impl std::fmt::Display) -> S3Error {
+    if persisted {
+        s3_error!(InternalError, "audit config persisted but runtime convergence failed: {}", error)
+    } else {
+        s3_error!(InternalError, "audit config unchanged but runtime convergence failed: {}", error)
+    }
+}
+
 pub(crate) async fn apply_audit_runtime_config(specs: &[AdminTargetSpec], config: Config) -> S3Result<()> {
     let has_targets = has_any_audit_targets(specs, &config);
 
@@ -107,15 +117,23 @@ where
             return Ok(());
         }
 
-        save_admin_server_config_snapshot(store, &config, &snapshot)
+        let persisted = save_admin_server_config_snapshot(store.clone(), &config, &snapshot)
             .await
-            .map(|_| ())
-            .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
+            .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?
+            .persisted();
+        drop(snapshot);
 
-        // Keep persistence and runtime publication in one detached, serialized
-        // mutation. Otherwise a cancelled caller or two concurrent updates can
-        // leave the persisted config and active audit generation disagreeing.
-        apply_audit_runtime_config(&specs, config).await
+        let read_store = store.clone();
+        with_runtime_config_reload_lock(async move {
+            let latest = with_admin_server_config_read_lock(store, move || read_existing_admin_server_config_no_lock(read_store))
+                .await
+                .map_err(|e| s3_error!(InternalError, "failed to lock server config for audit reload: {}", e))?
+                .map_err(|e| s3_error!(InternalError, "failed to read latest server config for audit reload: {}", e))?;
+
+            apply_audit_runtime_config(&specs, latest).await
+        })
+        .await
+        .map_err(|e| audit_config_convergence_error(persisted, e))
     })
     .await
 }
@@ -164,3 +182,154 @@ pub(crate) async fn remove_audit_target_config(specs: &[AdminTargetSpec], subsys
     })
     .await
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::admin::handlers::target_descriptor::admin_target_spec_from_builtin;
+    use crate::admin::runtime_sources::{IamInterface, KmsInterface};
+    use crate::admin::storage_api::config::save_admin_server_config;
+    use rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS;
+    use rustfs_config::server_config::KVS;
+    use rustfs_config::{ENABLE_KEY, EnableState, SCANNER_CYCLE, SCANNER_SUB_SYS, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR};
+    use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
+    use rustfs_kms::KmsServiceManager;
+    use rustfs_targets::catalog::builtin::builtin_audit_target_admin_descriptors;
+    use std::sync::Arc;
+    use std::time::Duration;
+    use tempfile::TempDir;
+
+    struct TestIam;
+
+    impl IamInterface for TestIam {
+        fn handle(&self) -> Arc<IamSys<ObjectStore>> {
+            unreachable!("audit config tests do not use IAM")
+        }
+
+        fn is_ready(&self) -> bool {
+            false
+        }
+    }
+
+    struct TestKms;
+
+    impl KmsInterface for TestKms {
+        fn handle(&self) -> Arc<KmsServiceManager> {
+            Arc::new(KmsServiceManager::new())
+        }
+    }
+
+    fn audit_specs() -> Vec<AdminTargetSpec> {
+        builtin_audit_target_admin_descriptors()
+            .into_iter()
+            .map(|descriptor| admin_target_spec_from_builtin(&descriptor))
+            .collect()
+    }
+
+    async fn wait_for_persisted_target(store: Arc<crate::admin::storage_api::runtime::ECStore>, subsystem: &str, target: &str) {
+        tokio::time::timeout(Duration::from_secs(30), async {
+            let mut poll = tokio::time::interval(Duration::from_millis(10));
+            loop {
+                poll.tick().await;
+                let config = read_admin_config_without_migrate(store.clone())
+                    .await
+                    .expect("read persisted server config");
+                if config.0.get(subsystem).is_some_and(|targets| targets.contains_key(target)) {
+                    return;
+                }
+            }
+        })
+        .await
+        .expect("config mutation should become durable");
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    #[serial_test::serial]
+    async fn audit_reload_reads_latest_durable_config_after_releasing_write_snapshot() {
+        let temp_dir = TempDir::new().expect("audit config temp dir");
+        let env = rustfs_test_utils::TestECStoreEnv::builder()
+            .base_dir(temp_dir.path())
+            .disk_count(1)
+            .init_bucket_metadata(false)
+            .build()
+            .await;
+        save_admin_server_config(env.ecstore.clone(), &Config::new())
+            .await
+            .expect("persist baseline server config");
+        let context = Arc::new(AppContext::new(env.ecstore.clone(), Arc::new(TestIam), Arc::new(TestKms)));
+
+        let (locked_tx, locked_rx) = tokio::sync::oneshot::channel();
+        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
+        let blocker = tokio::spawn(async move {
+            with_runtime_config_reload_lock(async move {
+                locked_tx.send(()).expect("signal runtime reload lock acquisition");
+                release_rx.await.expect("release runtime reload lock");
+                Ok(())
+            })
+            .await
+            .expect("runtime reload lock blocker");
+        });
+        locked_rx.await.expect("runtime reload lock should be held");
+
+        let older_context = context.clone();
+        let older = tokio::spawn(async move {
+            let specs = audit_specs();
+            update_audit_config_and_reload_for_context(Some(older_context.as_ref()), &specs, |config| {
+                config
+                    .0
+                    .entry(SCANNER_SUB_SYS.to_string())
+                    .or_default()
+                    .entry(DEFAULT_DELIMITER.to_string())
+                    .or_insert_with(KVS::new)
+                    .insert(SCANNER_CYCLE.to_string(), "15s".to_string());
+                true
+            })
+            .await
+        });
+        wait_for_persisted_target(env.ecstore.clone(), SCANNER_SUB_SYS, DEFAULT_DELIMITER).await;
+        assert!(!older.is_finished(), "audit runtime publication must wait for the shared reload lock");
+
+        let snapshot = read_admin_server_config_snapshot(env.ecstore.clone())
+            .await
+            .expect("read newer server config snapshot");
+        let mut latest = snapshot.config.clone();
+        let mut latest_target = KVS::new();
+        latest_target.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+        latest_target.insert(WEBHOOK_ENDPOINT.to_string(), "https://audit.invalid/hook".to_string());
+        latest_target.insert(
+            WEBHOOK_QUEUE_DIR.to_string(),
+            temp_dir.path().join("audit-queue").to_string_lossy().into_owned(),
+        );
+        latest
+            .0
+            .entry(AUDIT_WEBHOOK_SUB_SYS.to_string())
+            .or_default()
+            .insert("latest".to_string(), latest_target);
+        save_admin_server_config_snapshot(env.ecstore.clone(), &latest, &snapshot)
+            .await
+            .expect("persist newer audit config");
+        drop(snapshot);
+
+        release_tx.send(()).expect("release runtime reload blocker");
+        blocker.await.expect("runtime reload blocker task");
+        tokio::time::timeout(Duration::from_secs(30), older)
+            .await
+            .expect("older audit update should complete")
+            .expect("older audit update task should not panic")
+            .expect("older audit update should converge from the latest durable config");
+
+        let system = audit_system().expect("latest durable audit target should start the audit system");
+        let targets = system.list_targets().await;
+        assert!(targets.iter().any(|target| target.contains("latest")), "active targets: {targets:?}");
+        system.close().await.expect("audit system should stop after the test");
+    }
+
+    #[test]
+    fn audit_convergence_error_reports_durable_write_state() {
+        for (persisted, expected) in [(true, "audit config persisted"), (false, "audit config unchanged")] {
+            let error = audit_config_convergence_error(persisted, "injected failure");
+            assert!(error.to_string().contains(expected), "unexpected convergence error: {error}");
+            assert!(error.to_string().contains("injected failure"));
+        }
+    }
+}

rustfs/src/admin/handlers/config_admin.rs

@@ -17,18 +17,17 @@ use crate::admin::handlers::supervise_admin_mutation;
 use crate::admin::router::{AdminOperation, Operation, S3Router};
 use crate::admin::runtime_sources::{
     current_action_credentials, current_app_context, current_object_store_handle_for_context, current_server_config_for_context,
-    publish_server_config,
 };
 use crate::admin::service::config::{
-    CONFIG_WORKER_RELOAD_FAILURE_STATE, EVENT_CONFIG_WORKER_RELOAD_FAILED, FULL_CONFIG_WORKER_SUBSYSTEMS, LOG_COMPONENT_ADMIN,
-    LOG_SUBSYSTEM_CONFIG, PreparedRuntimeConfig, is_dynamic_config_subsystem, preflight_dynamic_config_reload,
-    prepare_server_config, reload_dynamic_config_runtime_state, reload_runtime_config_snapshot, signal_config_snapshot_reload,
-    signal_config_snapshot_reload_checked, signal_dynamic_config_reload_checked,
+    FULL_CONFIG_WORKER_SUBSYSTEMS, is_dynamic_config_subsystem, preflight_dynamic_config_reload, prepare_server_config,
+    publish_latest_runtime_config_snapshot, reload_dynamic_config_runtime_state, reload_runtime_config_snapshot,
+    signal_config_snapshot_reload, signal_config_snapshot_reload_checked, signal_dynamic_config_reload_checked,
 };
 use crate::admin::storage_api::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV};
 use crate::admin::storage_api::config::{
-    AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config,
-    read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_config, save_admin_server_config_snapshot,
+    AdminServerConfigSaveResult, AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config,
+    read_admin_config, read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_config,
+    save_admin_server_config_snapshot,
 };
 use crate::admin::storage_api::contract::list::ListOperations as _;
 use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body};
@@ -766,7 +765,10 @@ async fn load_server_config_snapshot_from_store() -> S3Result<AdminServerConfigS
         .map_err(Into::into)
 }
 
-async fn save_server_config_to_store(config: &ServerConfig, snapshot: &AdminServerConfigSnapshot) -> S3Result<bool> {
+async fn save_server_config_to_store(
+    config: &ServerConfig,
+    snapshot: &AdminServerConfigSnapshot,
+) -> S3Result<AdminServerConfigSaveResult> {
     let store = object_store()?;
     save_admin_server_config_snapshot(store, config, snapshot)
         .await
@@ -987,8 +989,8 @@ fn decode_config_history_snapshot(data: &[u8]) -> S3Result<ServerConfig> {
     Ok(config)
 }
 
-fn validate_restore_rollback_generation(current: &ServerConfig, restored: &ServerConfig) -> S3Result<()> {
-    if current == restored {
+fn validate_restore_rollback_generation(current: Option<Uuid>, committed: Option<Uuid>) -> S3Result<()> {
+    if current.is_some() && current == committed {
         Ok(())
     } else {
         Err(s3_error!(
@@ -1004,12 +1006,12 @@ async fn save_server_config_history_snapshot(config: &ServerConfig) -> S3Result<
     save_server_config_history(&sealed).await
 }
 
-async fn cleanup_failed_config_history_snapshot(restore_id: &str) {
+async fn cleanup_config_history_snapshot(restore_id: &str) {
     if let Err(err) = delete_server_config_history(restore_id).await {
         warn!(
             restore_id,
             error = %err,
-            "Failed to remove config history snapshot for an uncommitted mutation"
+            "Failed to remove config history snapshot"
         );
     }
 }
@@ -1843,23 +1845,6 @@ async fn reject_lost_config_transaction<T>(snapshot: AdminServerConfigSnapshot,
     ))
 }
 
-fn publish_prepared_config_snapshots(config: ServerConfig, prepared: PreparedRuntimeConfig) -> S3Result<()> {
-    prepared.publish_storage_class()?;
-    publish_server_config(config);
-    Ok(())
-}
-
-/// Re-apply local mutable worker families after a full-config replacement.
-/// Peers receive one full-snapshot signal after this returns; signaling each
-/// family here as well would recreate audit/scanner targets twice per peer.
-fn publish_notify_config_intent(
-    config: &ServerConfig,
-    sub_system: Option<&str>,
-) -> Option<rustfs_notify::NotificationLifecycleTransition> {
-    (sub_system.is_none() || sub_system.is_some_and(|sub_system| NOTIFY_SUB_SYSTEMS.contains(&sub_system)))
-        .then(|| rustfs_notify::ensure_live_events().publish_config(config.clone()))
-}
-
 fn config_preflight_subsystems(sub_system: Option<&str>) -> Vec<&str> {
     if let Some(sub_system) = sub_system {
         return is_dynamic_config_subsystem(sub_system)
@@ -1884,78 +1869,26 @@ async fn preflight_config_intent(sub_system: Option<&str>) -> S3Result<()> {
     Ok(())
 }
 
-async fn wait_notify_config_intent(transition: Option<rustfs_notify::NotificationLifecycleTransition>) -> S3Result<bool> {
-    let Some(transition) = transition else {
-        return Ok(false);
-    };
-    transition.wait().await.map_err(|err| {
-        warn!(error = %err, "Failed to apply local notification config");
-        s3_error!(InternalError, "failed to apply notification config")
-    })?;
-    Ok(true)
-}
-
-async fn reload_non_notify_dynamic_subsystems() -> Vec<String> {
-    let mut failures = Vec::new();
-    for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS {
-        if NOTIFY_SUB_SYSTEMS.contains(&sub_system) {
-            continue;
-        }
-        if reload_dynamic_config_runtime_state(sub_system).await.is_err() {
-            failures.push(format!("local {sub_system}"));
-            warn!(
-                event = EVENT_CONFIG_WORKER_RELOAD_FAILED,
-                component = LOG_COMPONENT_ADMIN,
-                subsystem = LOG_SUBSYSTEM_CONFIG,
-                config_subsystem = sub_system,
-                state = CONFIG_WORKER_RELOAD_FAILURE_STATE,
-                reason = "apply_failed",
-                "Published server config but failed to reload a local worker subsystem"
-            );
-        }
-    }
-    failures
-}
-
-fn finish_config_reconciliation(errors: Vec<String>) -> S3Result<()> {
+fn finish_config_reconciliation(errors: Vec<String>, persisted: bool) -> S3Result<()> {
     if errors.is_empty() {
         Ok(())
-    } else {
+    } else if persisted {
         Err(s3_error!(
             InternalError,
             "server config persisted but runtime convergence failed: {}",
             errors.join("; ")
         ))
+    } else {
+        Err(s3_error!(
+            InternalError,
+            "server config was unchanged but runtime convergence failed: {}",
+            errors.join("; ")
+        ))
     }
 }
 
-async fn reconcile_targeted_config(
-    sub_system: Option<String>,
-    storage_class_applied: bool,
-    notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
-) -> S3Result<bool> {
-    let mut errors = Vec::new();
-    let notify_applied = notify_transition.is_some();
-    if let Err(err) = wait_notify_config_intent(notify_transition).await {
-        warn!(error = %err, "Local notification config failed to converge");
-        errors.push("local notify".to_string());
-    }
-
-    let config_applied = if notify_applied {
-        if let Some(sub_system) = sub_system.as_deref()
-            && let Err(err) = signal_dynamic_config_reload_checked(sub_system).await
-        {
-            warn!(config_subsystem = sub_system, error = %err, "Peer config reload failed");
-            errors.push(format!("peer {sub_system}"));
-        }
-        true
-    } else if storage_class_applied {
-        if let Err(err) = signal_dynamic_config_reload_checked(STORAGE_CLASS_SUB_SYS).await {
-            warn!(error = %err, "Peer storage-class reload failed");
-            errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}"));
-        }
-        true
-    } else if let Some(sub_system) = sub_system.as_deref()
+async fn reconcile_targeted_config(sub_system: Option<String>, persisted: bool, mut errors: Vec<String>) -> S3Result<bool> {
+    let config_applied = if let Some(sub_system) = sub_system.as_deref()
         && is_dynamic_config_subsystem(sub_system)
     {
         let config_applied = match reload_dynamic_config_runtime_state(sub_system).await {
@@ -1979,21 +1912,19 @@ async fn reconcile_targeted_config(
         false
     };
 
-    finish_config_reconciliation(errors)?;
+    finish_config_reconciliation(errors, persisted)?;
     Ok(config_applied)
 }
 
-async fn reconcile_full_config(notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>) -> S3Result<()> {
-    let mut errors = Vec::new();
-    if let Err(err) = wait_notify_config_intent(notify_transition).await {
-        warn!(error = %err, "Local notification config failed to converge");
+async fn reconcile_full_config(persisted: bool, mut errors: Vec<String>) -> S3Result<()> {
+    if let Err(err) = reload_dynamic_config_runtime_state(NOTIFY_WEBHOOK_SUB_SYS).await {
+        warn!(error = %err, "Local notification config failed to converge from durable config");
         errors.push("local notify".to_string());
     }
     if let Err(err) = signal_dynamic_config_reload_checked(STORAGE_CLASS_SUB_SYS).await {
         warn!(error = %err, "Peer storage-class reload failed");
         errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}"));
     }
-    errors.extend(reload_non_notify_dynamic_subsystems().await);
     if let Err(err) = signal_dynamic_config_reload_checked(NOTIFY_WEBHOOK_SUB_SYS).await {
         warn!(error = %err, "Peer notification config reload failed");
         errors.push("peer notify".to_string());
@@ -2002,91 +1933,82 @@ async fn reconcile_full_config(notify_transition: Option<rustfs_notify::Notifica
         warn!(error = %err, "Peer config snapshot reload failed");
         errors.push("peer config snapshot".to_string());
     }
-    finish_config_reconciliation(errors)
+    finish_config_reconciliation(errors, persisted)
 }
 
 struct PersistedConfigTransaction {
     previous_config: ServerConfig,
     history_restore_id: Option<String>,
-    storage_class_applied: bool,
-    notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
+    persisted: bool,
+    committed_generation: Option<Uuid>,
 }
 
 async fn persist_server_config_transaction(
     config: ServerConfig,
-    prepared: PreparedRuntimeConfig,
     snapshot: AdminServerConfigSnapshot,
-    sub_system: Option<&str>,
 ) -> S3Result<PersistedConfigTransaction> {
     snapshot.ensure_lock_held().map_err(ApiError::from).map_err(S3Error::from)?;
     let previous_config = snapshot.config.clone();
     let history_restore_id = save_server_config_history_snapshot(&previous_config).await?;
     if snapshot.is_lock_lost() {
-        cleanup_failed_config_history_snapshot(&history_restore_id).await;
+        cleanup_config_history_snapshot(&history_restore_id).await;
         return reject_lost_config_transaction(snapshot, "before persistence").await;
     }
 
-    let persisted = match save_server_config_to_store(&config, &snapshot).await {
-        Ok(persisted) => persisted,
+    let save_result = match save_server_config_to_store(&config, &snapshot).await {
+        Ok(result) => result,
         Err(err) => {
-            cleanup_failed_config_history_snapshot(&history_restore_id).await;
+            cleanup_config_history_snapshot(&history_restore_id).await;
             return Err(err);
         }
     };
+    let persisted = save_result.persisted();
+    let committed_generation = save_result.generation();
     let history_restore_id = if persisted {
         Some(history_restore_id)
     } else {
-        cleanup_failed_config_history_snapshot(&history_restore_id).await;
+        cleanup_config_history_snapshot(&history_restore_id).await;
         None
     };
-    if snapshot.is_lock_lost() {
-        return reject_lost_config_transaction(snapshot, "after persistence").await;
-    }
-
-    let storage_class_applied = sub_system.is_none_or(|value| value == STORAGE_CLASS_SUB_SYS);
-    if storage_class_applied {
-        if let Err(err) = publish_prepared_config_snapshots(config.clone(), prepared) {
-            return Err(match history_restore_id.as_deref() {
-                Some(restore_id) => s3_error!(
-                    InternalError,
-                    "config persisted but runtime publish failed; recovery snapshot restoreId={}: {}",
-                    restore_id,
-                    err
-                ),
-                None => err,
-            });
-        }
-    } else {
-        publish_server_config(config.clone());
-    }
-    let notify_transition = publish_notify_config_intent(&config, sub_system);
-    if snapshot.is_lock_lost() {
-        return reject_lost_config_transaction(snapshot, "while publishing runtime snapshots").await;
-    }
-
     drop(snapshot);
     Ok(PersistedConfigTransaction {
         previous_config,
         history_restore_id,
-        storage_class_applied,
-        notify_transition,
+        persisted,
+        committed_generation,
     })
 }
 
+async fn reconcile_committed_config(sub_system: Option<String>, persisted: bool) -> S3Result<bool> {
+    let mut errors = Vec::new();
+    let publication_result = match sub_system.as_deref() {
+        Some(sub_system) => publish_latest_runtime_config_snapshot(sub_system).await,
+        None => reload_runtime_config_snapshot().await,
+    };
+    if let Err(err) = publication_result {
+        warn!(error = %err, "Failed to publish the latest durable server config");
+        errors.push("local config snapshot".to_string());
+    }
+
+    if sub_system.is_none() {
+        reconcile_full_config(persisted, errors).await?;
+        return Ok(false);
+    }
+
+    reconcile_targeted_config(sub_system, persisted, errors).await
+}
+
 async fn commit_server_config_transaction(
     config: ServerConfig,
-    prepared: PreparedRuntimeConfig,
     snapshot: AdminServerConfigSnapshot,
     sub_system: Option<String>,
 ) -> S3Result<bool> {
-    let transaction = persist_server_config_transaction(config, prepared, snapshot, sub_system.as_deref()).await?;
-
-    if sub_system.is_none() {
-        reconcile_full_config(transaction.notify_transition).await?;
-        return Ok(false);
+    let transaction = persist_server_config_transaction(config, snapshot).await?;
+    let result = reconcile_committed_config(sub_system, transaction.persisted).await;
+    match (result, transaction.history_restore_id.as_deref()) {
+        (Err(err), Some(restore_id)) => Err(s3_error!(InternalError, "{}; recovery snapshot restoreId={}", err, restore_id)),
+        (result, _) => result,
     }
-
-    reconcile_targeted_config(sub_system, transaction.storage_class_applied, transaction.notify_transition).await
 }
 
 pub struct GetConfigKVHandler {}
@@ -2127,8 +2049,8 @@ impl Operation for SetConfigKVHandler {
             let snapshot = load_server_config_snapshot_from_store().await?;
             let mut config = snapshot.config.clone();
             apply_set_directives(&mut config, &directives)?;
-            let prepared = prepare_server_config(&config, sub_system.as_deref()).await?;
-            commit_server_config_transaction(config, prepared, snapshot, sub_system).await
+            prepare_server_config(&config, sub_system.as_deref()).await?;
+            commit_server_config_transaction(config, snapshot, sub_system).await
         })
         .await?;
 
@@ -2155,8 +2077,8 @@ impl Operation for DelConfigKVHandler {
             let snapshot = load_server_config_snapshot_from_store().await?;
             let mut config = snapshot.config.clone();
             apply_delete_directives(&mut config, &directives);
-            let prepared = prepare_server_config(&config, sub_system.as_deref()).await?;
-            commit_server_config_transaction(config, prepared, snapshot, sub_system).await
+            prepare_server_config(&config, sub_system.as_deref()).await?;
+            commit_server_config_transaction(config, snapshot, sub_system).await
         })
         .await?;
 
@@ -2239,15 +2161,19 @@ impl Operation for RestoreConfigHistoryKVHandler {
         supervise_admin_mutation("config mutation", async move {
             preflight_config_intent(None).await?;
             let snapshot = load_server_config_snapshot_from_store().await?;
-            let prepared = prepare_server_config(&config, None).await?;
-            let restored_config = config.clone();
-            let transaction = persist_server_config_transaction(config, prepared, snapshot, None).await?;
-            let Err(restore_error) = reconcile_full_config(transaction.notify_transition).await else {
+            prepare_server_config(&config, None).await?;
+            let transaction = persist_server_config_transaction(config, snapshot).await?;
+            let Err(restore_error) = reconcile_committed_config(None, transaction.persisted).await else {
                 return Ok(());
             };
 
+            if !transaction.persisted {
+                return Err(restore_error);
+            }
+
             let previous_config = transaction.previous_config;
             let recovery_restore_id = transaction.history_restore_id;
+            let committed_generation = transaction.committed_generation;
             let recovery_reference = recovery_restore_id.as_deref().unwrap_or("not-created");
             let rollback_snapshot = match load_server_config_snapshot_from_store().await {
                 Ok(snapshot) => snapshot,
@@ -2261,7 +2187,9 @@ impl Operation for RestoreConfigHistoryKVHandler {
                     ));
                 }
             };
-            if let Err(rollback_error) = validate_restore_rollback_generation(&rollback_snapshot.config, &restored_config) {
+            if let Err(rollback_error) =
+                validate_restore_rollback_generation(rollback_snapshot.generation(), committed_generation)
+            {
                 return Err(s3_error!(
                     InternalError,
                     "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
@@ -2271,12 +2199,20 @@ impl Operation for RestoreConfigHistoryKVHandler {
                 ));
             }
 
-            let rollback_prepared = prepare_server_config(&previous_config, None).await?;
-            rollback_snapshot
+            if let Err(rollback_error) = prepare_server_config(&previous_config, None).await {
+                return Err(s3_error!(
+                    InternalError,
+                    "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
+                    restore_error,
+                    rollback_error,
+                    recovery_reference
+                ));
+            }
+            if let Err(rollback_error) = rollback_snapshot
                 .ensure_lock_held()
                 .map_err(ApiError::from)
-                .map_err(S3Error::from)?;
-            if let Err(rollback_error) = save_server_config_to_store(&previous_config, &rollback_snapshot).await {
+                .map_err(S3Error::from)
+            {
                 return Err(s3_error!(
                     InternalError,
                     "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
@@ -2285,10 +2221,9 @@ impl Operation for RestoreConfigHistoryKVHandler {
                     recovery_reference
                 ));
             }
-            if rollback_snapshot.is_lock_lost() {
-                if let Err(rollback_error) =
-                    reject_lost_config_transaction::<()>(rollback_snapshot, "after restore rollback persistence").await
-                {
+            let rollback_persisted = match save_server_config_to_store(&previous_config, &rollback_snapshot).await {
+                Ok(result) => result.persisted(),
+                Err(rollback_error) => {
                     return Err(s3_error!(
                         InternalError,
                         "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
@@ -2297,46 +2232,20 @@ impl Operation for RestoreConfigHistoryKVHandler {
                         recovery_reference
                     ));
                 }
-                return Err(s3_error!(InternalError, "restore rollback lock-loss handling returned unexpectedly"));
-            }
-
-            if let Err(rollback_error) = publish_prepared_config_snapshots(previous_config.clone(), rollback_prepared) {
-                return Err(s3_error!(
-                    InternalError,
-                    "config restore failed: {}; automatic rollback publish failed: {}; recovery snapshot restoreId={}",
-                    restore_error,
-                    rollback_error,
-                    recovery_reference
-                ));
-            }
-            let rollback_transition = publish_notify_config_intent(&previous_config, None);
-            if rollback_snapshot.is_lock_lost() {
-                if let Err(rollback_error) =
-                    reject_lost_config_transaction::<()>(rollback_snapshot, "while publishing restore rollback").await
-                {
-                    return Err(s3_error!(
-                        InternalError,
-                        "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
-                        restore_error,
-                        rollback_error,
-                        recovery_reference
-                    ));
-                }
-                return Err(s3_error!(InternalError, "restore rollback lock-loss handling returned unexpectedly"));
-            }
+            };
             drop(rollback_snapshot);
 
-            if let Err(rollback_error) = reconcile_full_config(rollback_transition).await {
+            if let Err(rollback_error) = reconcile_committed_config(None, rollback_persisted).await {
                 return Err(s3_error!(
                     InternalError,
-                    "config restore failed: {}; persisted rollback did not converge: {}; recovery snapshot restoreId={}",
+                    "config restore failed: {}; automatic rollback convergence failed: {}; recovery snapshot restoreId={}",
                     restore_error,
                     rollback_error,
                     recovery_reference
                 ));
             }
-            if let Some(recovery_restore_id) = recovery_restore_id.as_deref() {
-                cleanup_failed_config_history_snapshot(recovery_restore_id).await;
+            if rollback_persisted && let Some(recovery_restore_id) = recovery_restore_id.as_deref() {
+                cleanup_config_history_snapshot(recovery_restore_id).await;
             }
             Err(s3_error!(InternalError, "config restore failed and was rolled back: {}", restore_error))
         })
@@ -2377,8 +2286,8 @@ impl Operation for SetConfigHandler {
             let snapshot = load_server_config_snapshot_from_store().await?;
             let mut config = ServerConfig::new();
             apply_set_directives(&mut config, &directives)?;
-            let prepared = prepare_server_config(&config, None).await?;
-            commit_server_config_transaction(config, prepared, snapshot, None).await?;
+            prepare_server_config(&config, None).await?;
+            commit_server_config_transaction(config, snapshot, None).await?;
             Ok(())
         })
         .await?;
@@ -2408,6 +2317,23 @@ mod tests {
         );
     }
 
+    #[test]
+    fn reconciliation_error_reports_whether_config_was_persisted() {
+        let persisted = finish_config_reconciliation(vec!["local config snapshot".to_string()], true)
+            .expect_err("committed config convergence failure must be reported");
+        let unchanged = finish_config_reconciliation(vec!["local config snapshot".to_string()], false)
+            .expect_err("unchanged config convergence failure must be reported");
+
+        assert_eq!(
+            persisted.message(),
+            Some("server config persisted but runtime convergence failed: local config snapshot")
+        );
+        assert_eq!(
+            unchanged.message(),
+            Some("server config was unchanged but runtime convergence failed: local config snapshot")
+        );
+    }
+
     #[test]
     fn tokenize_config_line_handles_quotes_and_escapes() {
         let tokens = tokenize_config_line(r#"identity_openid client_id="console app" client_secret="s3cr\"et" enable=on"#)
@@ -3191,23 +3117,32 @@ notify_webhook:secondary endpoint="https://secondary.example" auth_token="second
     }
 
     #[test]
-    fn restore_rollback_generation_rejects_concurrent_config_change() {
-        crate::admin::storage_api::config::init_admin_config_defaults();
-        let restored = ServerConfig::new();
-        let mut concurrent = restored.clone();
-        apply_set_directives(
-            &mut concurrent,
-            &parse_config_directives(r#"identity_openid client_id="concurrent-client""#, false).expect("parse concurrent"),
-        )
-        .expect("apply concurrent");
+    fn restore_rollback_generation_accepts_committed_write_identity() {
+        let committed = Uuid::from_u128(1);
+        validate_restore_rollback_generation(Some(committed), Some(committed)).expect("matching committed generation");
+    }
 
-        validate_restore_rollback_generation(&restored, &restored).expect("unchanged restore generation");
-        let error = validate_restore_rollback_generation(&concurrent, &restored)
-            .expect_err("concurrent change must fence automatic rollback");
+    #[test]
+    fn restore_rollback_generation_rejects_aba_with_matching_config_content() {
+        let error = validate_restore_rollback_generation(Some(Uuid::from_u128(2)), Some(Uuid::from_u128(1)))
+            .expect_err("a later generation must fence automatic rollback even when config content matches");
         assert_eq!(error.code(), &S3ErrorCode::InvalidRequest);
         assert!(error.to_string().contains("concurrent configuration change"));
     }
 
+    #[test]
+    fn restore_rollback_generation_rejects_missing_write_identity() {
+        for (current, committed) in [
+            (None, Some(Uuid::from_u128(1))),
+            (Some(Uuid::from_u128(1)), None),
+            (None, None),
+        ] {
+            let error = validate_restore_rollback_generation(current, committed)
+                .expect_err("missing generation metadata must fence automatic rollback");
+            assert_eq!(error.code(), &S3ErrorCode::InvalidRequest);
+        }
+    }
+
     #[test]
     fn legacy_directive_history_is_rejected_without_being_applied_as_a_snapshot() {
         let error = decode_config_history_snapshot(b"identity_openid client_id=\"legacy\"")

rustfs/src/admin/service/config.rs

@@ -16,7 +16,9 @@ use crate::admin::runtime_sources::{
     AppContext, current_app_context, current_notification_system_for_context, current_object_store_handle_for_context,
     publish_server_config, publish_storage_class_config,
 };
-use crate::admin::storage_api::config::{STORAGE_CLASS_SUB_SYS, read_admin_config_without_migrate, storageclass};
+use crate::admin::storage_api::config::{
+    STORAGE_CLASS_SUB_SYS, read_existing_admin_server_config_no_lock, storageclass, with_admin_server_config_read_lock,
+};
 use crate::admin::storage_api::contract::admin::StorageAdminApi;
 use crate::admin::storage_api::runtime::ECStore;
 use crate::server::{
@@ -43,6 +45,25 @@ use url::Url;
 
 static RUNTIME_CONFIG_RELOAD_MUTEX: AsyncMutex<()> = AsyncMutex::const_new(());
 
+// Runtime publication lock order: reload mutex -> server-config local lock ->
+// transaction lock -> config object lock.
+pub(crate) async fn with_runtime_config_reload_lock<Fut, T>(operation: Fut) -> S3Result<T>
+where
+    Fut: Future<Output = S3Result<T>> + Send + 'static,
+    T: Send + 'static,
+{
+    let reload_guard = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await;
+    tokio::spawn(async move {
+        let _reload_guard = reload_guard;
+        operation.await
+    })
+    .await
+    .map_err(|err| {
+        let outcome = if err.is_cancelled() { "cancelled" } else { "panicked" };
+        internal_error(format!("runtime config reload task {outcome}"))
+    })?
+}
+
 pub fn is_dynamic_config_subsystem(sub_system: &str) -> bool {
     NOTIFY_SUB_SYSTEMS.contains(&sub_system)
         || matches!(
@@ -121,11 +142,6 @@ impl PreparedRuntimeConfig {
     fn publish_storage_class_for_context(self, context: Option<&AppContext>) -> S3Result<()> {
         self.publish_storage_class_for_context_with(context, publish_storage_class_config)
     }
-
-    pub(crate) fn publish_storage_class(self) -> S3Result<()> {
-        let context = current_app_context();
-        self.publish_storage_class_for_context(context.as_deref())
-    }
 }
 
 fn publish_server_config_for_context(context: Option<&AppContext>, config: ServerConfig) {
@@ -396,7 +412,18 @@ pub async fn apply_dynamic_config_for_subsystem(config: &ServerConfig, sub_syste
 }
 
 pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> {
-    let _reload_guard = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await;
+    let context = context.cloned();
+    let sub_system = sub_system.to_owned();
+    with_runtime_config_reload_lock(async move {
+        reload_dynamic_config_runtime_state_under_reload_lock_for_context(context.as_ref(), &sub_system).await
+    })
+    .await
+}
+
+async fn reload_dynamic_config_runtime_state_under_reload_lock_for_context(
+    context: Option<&AppContext>,
+    sub_system: &str,
+) -> S3Result<()> {
     if sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM {
         let store = resolve_runtime_config_store_for_context(context)?;
         let notify_result = reconcile_event_notifier_from_store(store).await;
@@ -414,18 +441,54 @@ pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&Ap
     }
 
     let store = resolve_runtime_config_store_for_context(context)?;
-    let config = read_admin_config_without_migrate(store).await.map_err(|err| {
-        warn!("peer reload_dynamic_config: failed to load server config for {sub_system}: {err}");
-        internal_error(format!("failed to load server config: {err}"))
-    })?;
+    let read_store = store.clone();
+    let publication_context = context.cloned();
+    let publication_sub_system = sub_system.to_owned();
+    let notify_config = with_admin_server_config_read_lock(store, move || async move {
+        let config = read_existing_admin_server_config_no_lock(read_store).await.map_err(|err| {
+            warn!("peer reload_dynamic_config: failed to load server config for {publication_sub_system}: {err}");
+            internal_error(format!("failed to load server config: {err}"))
+        })?;
+        let prepared = prepare_server_config_for_context(publication_context.as_ref(), &config, Some(&publication_sub_system))
+            .await
+            .map_err(|err| {
+                if publication_sub_system == STORAGE_CLASS_SUB_SYS {
+                    internal_error(format!("failed to apply storage class config: {err}"))
+                } else {
+                    err
+                }
+            })?;
 
-    if matches!(sub_system, SCANNER_SUB_SYS | HEAL_SUB_SYS) {
-        validate_server_config_for_context(context, &config, Some(sub_system)).await?;
-        // Scanner cycles refresh from the process-wide server config before
-        // each pass. Publish the same validated snapshot first so that refresh
-        // cannot overwrite this peer's dynamic scanner update with stale data.
-        publish_server_config_for_context(context, config.clone());
-    }
+        if publication_sub_system == STORAGE_CLASS_SUB_SYS {
+            prepared.publish_storage_class_for_context(publication_context.as_ref())?;
+            return Ok::<Option<ServerConfig>, S3Error>(None);
+        }
+        if NOTIFY_SUB_SYSTEMS.contains(&publication_sub_system.as_str()) {
+            return Ok(Some(config));
+        }
+        if matches!(publication_sub_system.as_str(), SCANNER_SUB_SYS | HEAL_SUB_SYS) {
+            publish_server_config_for_context(publication_context.as_ref(), config.clone());
+        }
+        apply_dynamic_config_for_subsystem_for_context(publication_context.as_ref(), &config, &publication_sub_system)
+            .await
+            .inspect_err(|_| {
+                warn!(
+                    config_subsystem = publication_sub_system,
+                    reason = "apply_failed",
+                    "Peer dynamic config apply failed"
+                );
+            })?;
+        Ok(None)
+    })
+    .await
+    .map_err(|err| {
+        warn!("peer reload_dynamic_config: failed to acquire server config publication fence for {sub_system}: {err}");
+        internal_error(format!("failed to lock server config: {err}"))
+    })??;
+
+    let Some(config) = notify_config else {
+        return Ok(());
+    };
     apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system)
         .await
         .inspect_err(|_| {
@@ -439,23 +502,16 @@ pub async fn reload_dynamic_config_runtime_state(sub_system: &str) -> S3Result<(
     reload_dynamic_config_runtime_state_for_context(context.as_deref(), sub_system).await
 }
 
-async fn reload_runtime_config_snapshot_with<ReadFuture, Prepare, PrepareFuture, Publish, ApplyWorkers, ApplyWorkersFuture>(
-    read: ReadFuture,
-    prepare: Prepare,
-    publish: Publish,
+async fn reload_runtime_config_snapshot_with<PublishFuture, ApplyWorkers, ApplyWorkersFuture>(
+    publish_snapshot: PublishFuture,
     apply_workers: ApplyWorkers,
 ) -> S3Result<()>
 where
-    ReadFuture: Future<Output = S3Result<ServerConfig>>,
-    Prepare: FnOnce(ServerConfig) -> PrepareFuture,
-    PrepareFuture: Future<Output = S3Result<(ServerConfig, PreparedRuntimeConfig)>>,
-    Publish: FnOnce(&ServerConfig, PreparedRuntimeConfig) -> S3Result<()>,
+    PublishFuture: Future<Output = S3Result<ServerConfig>>,
     ApplyWorkers: FnOnce(ServerConfig) -> ApplyWorkersFuture,
     ApplyWorkersFuture: Future<Output = S3Result<()>>,
 {
-    let config = read.await?;
-    let (config, prepared) = prepare(config).await?;
-    publish(&config, prepared)?;
+    let config = publish_snapshot.await?;
 
     // Worker reloads mutate live state and have no rollback contract, so the
     // validated snapshots stay published. The RPC still reports convergence
@@ -474,55 +530,105 @@ where
     Ok(())
 }
 
-pub async fn reload_runtime_config_snapshot_for_context(context: Option<&AppContext>) -> S3Result<()> {
-    let _reload_guard = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await;
+async fn publish_latest_runtime_config_snapshot_under_reload_lock_for_context<ApplyWorkers, ApplyWorkersFuture>(
+    context: Option<&AppContext>,
+    sub_system: Option<&str>,
+    apply_workers: ApplyWorkers,
+) -> S3Result<()>
+where
+    ApplyWorkers: FnOnce(ServerConfig) -> ApplyWorkersFuture + Send + 'static,
+    ApplyWorkersFuture: Future<Output = S3Result<()>> + Send + 'static,
+{
     let store = resolve_runtime_config_store_for_context(context)?;
+    let read_store = store.clone();
+    let publication_context = context.cloned();
+    let publication_sub_system = sub_system.map(str::to_owned);
 
-    reload_runtime_config_snapshot_with(
-        async move {
-            read_admin_config_without_migrate(store).await.map_err(|err| {
-                warn!("peer reload_runtime_config_snapshot: failed to load server config: {err}");
-                internal_error(format!("failed to load server config: {err}"))
-            })
-        },
-        |config| async move {
-            let prepared = prepare_server_config_for_context(context, &config, None).await.map_err(|_| {
-                warn!("peer reload_runtime_config_snapshot: failed to prepare server config");
-                internal_error("failed to prepare server config")
-            })?;
-            Ok((config, prepared))
-        },
-        |config, prepared| {
-            prepared.publish_storage_class_for_context(context)?;
-            publish_server_config_for_context(context, config.clone());
-            Ok(())
-        },
-        |config| async move {
-            let mut failures = Vec::new();
-            for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS {
-                if apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system)
-                    .await
-                    .is_err()
+    with_admin_server_config_read_lock(store, move || async move {
+        let config = read_existing_admin_server_config_no_lock(read_store).await.map_err(|err| {
+            warn!("runtime config publication failed to load server config: {err}");
+            internal_error(format!("failed to load server config: {err}"))
+        })?;
+        let prepared =
+            prepare_server_config_for_context(publication_context.as_ref(), &config, publication_sub_system.as_deref())
+                .await
+                .map_err(|err| match publication_sub_system.as_deref() {
+                    None => {
+                        warn!("peer reload_runtime_config_snapshot: failed to prepare server config");
+                        internal_error("failed to prepare server config")
+                    }
+                    Some(STORAGE_CLASS_SUB_SYS) => internal_error(format!("failed to apply storage class config: {err}")),
+                    Some(_) => err,
+                })?;
+        reload_runtime_config_snapshot_with(
+            async move {
+                if publication_sub_system
+                    .as_deref()
+                    .is_none_or(|sub_system| sub_system == STORAGE_CLASS_SUB_SYS)
                 {
-                    failures.push(sub_system);
-                    warn!(
-                        event = EVENT_CONFIG_WORKER_RELOAD_FAILED,
-                        component = LOG_COMPONENT_ADMIN,
-                        subsystem = LOG_SUBSYSTEM_CONFIG,
-                        config_subsystem = sub_system,
-                        state = CONFIG_WORKER_RELOAD_FAILURE_STATE,
-                        reason = "apply_failed",
-                        "Peer runtime config snapshot was published but a subsystem worker reload failed"
-                    );
+                    prepared.publish_storage_class_for_context(publication_context.as_ref())?;
                 }
+                publish_server_config_for_context(publication_context.as_ref(), config.clone());
+                Ok(config)
+            },
+            apply_workers,
+        )
+        .await
+    })
+    .await
+    .map_err(|err| {
+        warn!("runtime config publication failed to acquire server config publication fence: {err}");
+        internal_error(format!("failed to lock server config: {err}"))
+    })?
+}
+
+pub(crate) async fn publish_latest_runtime_config_snapshot(sub_system: &str) -> S3Result<()> {
+    let context = current_app_context();
+    let sub_system = sub_system.to_owned();
+    with_runtime_config_reload_lock(async move {
+        publish_latest_runtime_config_snapshot_under_reload_lock_for_context(context.as_deref(), Some(&sub_system), |_| async {
+            Ok(())
+        })
+        .await
+    })
+    .await
+}
+
+pub async fn reload_runtime_config_snapshot_for_context(context: Option<&AppContext>) -> S3Result<()> {
+    let context = context.cloned();
+    with_runtime_config_reload_lock(async move {
+        reload_runtime_config_snapshot_under_reload_lock_for_context(context.as_ref()).await
+    })
+    .await
+}
+
+async fn reload_runtime_config_snapshot_under_reload_lock_for_context(context: Option<&AppContext>) -> S3Result<()> {
+    let worker_context = context.cloned();
+    publish_latest_runtime_config_snapshot_under_reload_lock_for_context(context, None, move |config| async move {
+        let mut failures = Vec::new();
+        for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS {
+            if apply_dynamic_config_for_subsystem_for_context(worker_context.as_ref(), &config, sub_system)
+                .await
+                .is_err()
+            {
+                failures.push(sub_system);
+                warn!(
+                    event = EVENT_CONFIG_WORKER_RELOAD_FAILED,
+                    component = LOG_COMPONENT_ADMIN,
+                    subsystem = LOG_SUBSYSTEM_CONFIG,
+                    config_subsystem = sub_system,
+                    state = CONFIG_WORKER_RELOAD_FAILURE_STATE,
+                    reason = "apply_failed",
+                    "Peer runtime config snapshot was published but a subsystem worker reload failed"
+                );
             }
-            if failures.is_empty() {
-                Ok(())
-            } else {
-                Err(internal_error(format!("runtime worker reload failed: {}", failures.join("; "))))
-            }
-        },
-    )
+        }
+        if failures.is_empty() {
+            Ok(())
+        } else {
+            Err(internal_error(format!("runtime worker reload failed: {}", failures.join("; "))))
+        }
+    })
     .await
 }
 
@@ -699,6 +805,77 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn runtime_reload_waiter_cancellation_does_not_leave_queued_work() {
+        let _blocker = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await;
+        let (operation_started_tx, mut operation_started_rx) = tokio::sync::oneshot::channel();
+        let mut reload = Box::pin(with_runtime_config_reload_lock(async move {
+            let _ = operation_started_tx.send(());
+            Ok(())
+        }));
+
+        poll_fn(|cx| match reload.as_mut().poll(cx) {
+            Poll::Pending => Poll::Ready(()),
+            Poll::Ready(_) => panic!("reload must wait while the mutex is held"),
+        })
+        .await;
+        drop(reload);
+
+        assert!(
+            matches!(operation_started_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Closed)),
+            "cancelling a queued reload must drop its work instead of detaching it"
+        );
+    }
+
+    #[tokio::test]
+    async fn runtime_reload_lock_survives_waiter_cancellation() {
+        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
+        let (completed_tx, completed_rx) = tokio::sync::oneshot::channel();
+
+        let waiter = tokio::spawn(async move {
+            with_runtime_config_reload_lock(async move {
+                started_tx.send(()).expect("signal runtime reload start");
+                release_rx.await.expect("release supervised runtime reload");
+                completed_tx.send(()).expect("signal runtime reload completion");
+                Ok(())
+            })
+            .await
+        });
+
+        started_rx.await.expect("supervised runtime reload started");
+        waiter.abort();
+        assert!(waiter.await.expect_err("reload waiter should be cancelled").is_cancelled());
+
+        let (contender_polled_tx, contender_polled_rx) = tokio::sync::oneshot::channel();
+        let (contender_entered_tx, mut contender_entered_rx) = tokio::sync::oneshot::channel();
+        let contender = tokio::spawn(async move {
+            let mut lock = Box::pin(RUNTIME_CONFIG_RELOAD_MUTEX.lock());
+            let mut contender_polled_tx = Some(contender_polled_tx);
+            let _guard = poll_fn(|cx| {
+                if let Some(tx) = contender_polled_tx.take() {
+                    let _ = tx.send(());
+                }
+                lock.as_mut().poll(cx)
+            })
+            .await;
+            let _ = contender_entered_tx.send(());
+        });
+
+        contender_polled_rx.await.expect("contending reload should be polled");
+        assert!(
+            matches!(contender_entered_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)),
+            "a cancelled waiter must not release the reload mutex while its detached reload is still running"
+        );
+
+        release_tx.send(()).expect("release detached runtime reload");
+        completed_rx.await.expect("detached runtime reload should complete");
+        contender_entered_rx
+            .await
+            .expect("contending reload should enter after completion");
+        contender.await.expect("contending reload task should not panic");
+    }
+
     #[tokio::test]
     async fn checked_scanner_reload_reports_unreachable_peer() {
         let temp_dir = TempDir::new().expect("scanner reload temp dir");
@@ -1221,6 +1398,123 @@ mod tests {
         .await;
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    #[serial_test::serial(storage_class_env)]
+    async fn full_reload_keeps_worker_apply_inside_durable_read_fence() {
+        temp_env::async_with_vars(
+            [
+                (storageclass::STANDARD_ENV, None::<&str>),
+                (storageclass::RRS_ENV, None::<&str>),
+                (storageclass::OPTIMIZE_ENV, None::<&str>),
+                (storageclass::INLINE_BLOCK_ENV, None::<&str>),
+            ],
+            async {
+                let fixture = runtime_config_reload_fixture().await;
+                let older = scanner_server_config("41");
+                save_admin_server_config(fixture.context.object_store(), &older)
+                    .await
+                    .expect("persist older scanner config");
+
+                let (worker_entered_tx, worker_entered_rx) = tokio::sync::oneshot::channel();
+                let (release_worker_tx, release_worker_rx) = tokio::sync::oneshot::channel();
+                let reload_context = fixture.context.clone();
+                let reload = tokio::spawn(async move {
+                    publish_latest_runtime_config_snapshot_under_reload_lock_for_context(
+                        Some(&reload_context),
+                        None,
+                        move |config| async move {
+                            worker_entered_tx.send(config).expect("signal runtime config worker entry");
+                            release_worker_rx.await.expect("release runtime config worker");
+                            Ok(())
+                        },
+                    )
+                    .await
+                });
+
+                let worker_config = tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, worker_entered_rx)
+                    .await
+                    .expect("worker apply should enter")
+                    .expect("worker apply entry signal should be delivered");
+                assert_eq!(
+                    worker_config
+                        .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER)
+                        .expect("older scanner config should be loaded")
+                        .get(SCANNER_CYCLE),
+                    "41"
+                );
+
+                let latest = scanner_server_config("71");
+                let writer_store = fixture.context.object_store();
+                let (writer_polled_tx, writer_polled_rx) = tokio::sync::oneshot::channel();
+                let (writer_entered_tx, mut writer_entered_rx) = tokio::sync::oneshot::channel();
+                let writer = tokio::spawn(async move {
+                    let transaction_store = writer_store.clone();
+                    let transaction = with_admin_server_config_write_lock(writer_store, move || async move {
+                        writer_entered_tx.send(()).expect("signal writer entry");
+                        save_admin_server_config_no_lock(transaction_store, &latest).await
+                    });
+                    tokio::pin!(transaction);
+                    let mut writer_polled_tx = Some(writer_polled_tx);
+                    poll_fn(|cx| match transaction.as_mut().poll(cx) {
+                        Poll::Pending => {
+                            if let Some(tx) = writer_polled_tx.take() {
+                                let _ = tx.send(());
+                            }
+                            Poll::Ready(())
+                        }
+                        Poll::Ready(_) => panic!("writer entered before the worker apply released its read fence"),
+                    })
+                    .await;
+                    transaction
+                        .await
+                        .expect("writer should acquire the server-config locks")
+                        .expect("writer should persist the latest config");
+                });
+
+                tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, writer_polled_rx)
+                    .await
+                    .expect("writer should be polled")
+                    .expect("writer poll signal should be delivered");
+                assert!(
+                    matches!(writer_entered_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)),
+                    "a server-config writer must wait until the old worker apply completes"
+                );
+
+                release_worker_tx.send(()).expect("release worker apply");
+                tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, async {
+                    reload
+                        .await
+                        .expect("full reload task should not panic")
+                        .expect("full reload should finish");
+                    writer.await.expect("writer task should not panic");
+                })
+                .await
+                .expect("reload and writer should finish");
+
+                reload_runtime_config_snapshot_for_context(Some(&fixture.context))
+                    .await
+                    .expect("a later full reload should publish the latest durable config");
+                let snapshot = fixture
+                    .server_snapshot
+                    .lock()
+                    .expect("server config result lock")
+                    .clone()
+                    .expect("latest server config should be published");
+                assert_eq!(
+                    snapshot
+                        .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER)
+                        .expect("latest scanner config should be present")
+                        .get(SCANNER_CYCLE),
+                    "71"
+                );
+                assert_eq!(rustfs_scanner::scanner_runtime_config_status().cycle_interval_seconds.value, 71);
+
+                rustfs_scanner::apply_scanner_runtime_config(&ServerConfig::new()).expect("restore scanner runtime defaults");
+            },
+        )
+        .await;
+    }
+
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
     async fn peer_full_reload_rejects_later_pool_without_publishing() {
@@ -1251,11 +1545,9 @@ mod tests {
         let worker_events = events.clone();
 
         let err = reload_runtime_config_snapshot_with(
-            async { Ok(ServerConfig::new()) },
-            |config| async { Ok((config, PreparedRuntimeConfig::default())) },
-            move |_config, _prepared| {
+            async move {
                 publish_events.lock().expect("reload event lock").push("publish");
-                Ok(())
+                Ok(ServerConfig::new())
             },
             move |_config| async move {
                 let mut events = worker_events.lock().expect("reload event lock");

rustfs/src/admin/storage_api.rs

@@ -644,12 +644,17 @@ pub(crate) async fn read_admin_config_without_migrate(api: Arc<ECStore>) -> Resu
     ecstore_config::com::read_config_without_migrate(api).await
 }
 
+pub(crate) async fn read_existing_admin_server_config_no_lock(api: Arc<ECStore>) -> Result<rustfs_config::server_config::Config> {
+    ecstore_config::com::read_existing_server_config_no_lock(api).await
+}
+
 #[cfg(test)]
 pub(crate) async fn read_admin_config_without_migrate_no_lock(api: Arc<ECStore>) -> Result<rustfs_config::server_config::Config> {
     ecstore_config::com::read_config_without_migrate_no_lock(api).await
 }
 
 pub(crate) type AdminServerConfigSnapshot = ecstore_config::com::ServerConfigSnapshot;
+pub(crate) type AdminServerConfigSaveResult = ecstore_config::com::ServerConfigSaveResult;
 
 pub(crate) async fn save_admin_config(api: Arc<ECStore>, file: &str, data: Vec<u8>) -> Result<()> {
     ecstore_config::com::save_config(api, file, data).await
@@ -690,8 +695,17 @@ pub(crate) async fn save_admin_server_config_snapshot(
     api: Arc<ECStore>,
     cfg: &rustfs_config::server_config::Config,
     snapshot: &AdminServerConfigSnapshot,
-) -> Result<bool> {
-    ecstore_config::com::save_server_config_snapshot(api, cfg, snapshot).await
+) -> Result<AdminServerConfigSaveResult> {
+    ecstore_config::com::save_server_config_snapshot_with_generation(api, cfg, snapshot).await
+}
+
+pub(crate) async fn with_admin_server_config_read_lock<F, Fut, T>(api: Arc<ECStore>, operation: F) -> Result<T>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    ecstore_config::com::with_server_config_read_lock(api, operation).await
 }
 
 pub(crate) fn init_admin_config_defaults() {
@@ -793,9 +807,10 @@ pub(crate) mod cluster {
 pub(crate) mod config {
     pub(crate) use super::storageclass;
     pub(crate) use super::{
-        AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, init_admin_config_defaults,
-        read_admin_config, read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_config,
-        save_admin_server_config_snapshot,
+        AdminServerConfigSaveResult, AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config,
+        init_admin_config_defaults, read_admin_config, read_admin_config_without_migrate, read_admin_server_config_snapshot,
+        read_existing_admin_server_config_no_lock, save_admin_config, save_admin_server_config_snapshot,
+        with_admin_server_config_read_lock,
     };
     #[cfg(test)]
     pub(crate) use super::{