Centralize lifecycle state updates and fix systemd running status (#2567)

rustfs/src/admin/handlers/policies.rs

@@ -641,8 +641,8 @@ fn build_policy_mappings(
     }
 
     let mut results: Vec<PolicyEntities> = policy_map
-        .into_iter()
-        .filter_map(|(_, mut mapping)| {
+        .into_values()
+        .filter_map(|mut mapping| {
             if !requested_policies.is_empty() && !requested_policies.iter().any(|policy| policy == &mapping.policy) {
                 return None;
             }

rustfs/src/embedded.rs

@@ -28,8 +28,8 @@
 //!     let port = find_available_port()?;
 //!     let server = RustFSServerBuilder::new()
 //!         .address(format!("127.0.0.1:{port}"))
-//!         .access_key("minioadmin")
-//!         .secret_key("minioadmin")
+//!         .access_key("rustfsadmin")
+//!         .secret_key("rustfsadmin")
 //!         .build()
 //!         .await?;
 //!
@@ -49,10 +49,7 @@
 use crate::app::context::{AppContext, init_global_app_context};
 use crate::config::Config;
 use crate::init::{add_bucket_notification_configuration, init_buffer_profile_system, init_kms_system};
-use crate::server::{
-    ServiceState, ServiceStateManager, init_event_notifier, shutdown_event_notifier, start_audit_system, start_http_server,
-    stop_audit_system,
-};
+use crate::server::{init_event_notifier, shutdown_event_notifier, start_audit_system, start_http_server, stop_audit_system};
 use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr};
 use rustfs_credentials::init_global_action_credentials;
 use rustfs_ecstore::store::init_lock_clients;
@@ -166,7 +163,7 @@ impl RustFSServerBuilder {
     ///
     /// Defaults:
     /// - address: `"127.0.0.1:9000"`
-    /// - access_key / secret_key: `"minioadmin"`
+    /// - access_key / secret_key: `"rustfsadmin"`
     /// - region: `"us-east-1"`
     /// - A temporary directory is created automatically for data storage
     ///
@@ -175,10 +172,10 @@ impl RustFSServerBuilder {
     pub fn new() -> Self {
         Self {
             address: "127.0.0.1:9000".to_string(),
-            access_key: "minioadmin".to_string(),
-            secret_key: "minioadmin".to_string(),
+            access_key: rustfs_credentials::DEFAULT_ACCESS_KEY.to_string(),
+            secret_key: rustfs_credentials::DEFAULT_SECRET_KEY.to_string(),
             volumes: Vec::new(),
-            region: "us-east-1".to_string(),
+            region: rustfs_config::RUSTFS_REGION.to_string(),
         }
     }
 
@@ -196,13 +193,13 @@ impl RustFSServerBuilder {
         self
     }
 
-    /// Set the S3 access key (default: `"minioadmin"`).
+    /// Set the S3 access key (default: `"rustfsadmin"`).
     pub fn access_key(mut self, key: impl Into<String>) -> Self {
         self.access_key = key.into();
         self
     }
 
-    /// Set the S3 secret key (default: `"minioadmin"`).
+    /// Set the S3 secret key (default: `"rustfsadmin"`).
     pub fn secret_key(mut self, key: impl Into<String>) -> Self {
         self.secret_key = key.into();
         self
@@ -355,13 +352,11 @@ impl RustFSServerBuilder {
 
         // Service state.
         let readiness = Arc::new(GlobalReadiness::new());
-        let state_manager = ServiceStateManager::new();
-        state_manager.update(ServiceState::Starting);
 
         // Start HTTP server.
         let mut s3_config = config.clone();
         s3_config.console_enable = false;
-        let (shutdown_tx, bound_addr) = start_http_server(&s3_config, state_manager.clone(), readiness.clone()).await?;
+        let (shutdown_tx, bound_addr) = start_http_server(&s3_config, readiness.clone()).await?;
         let ctx = CancellationToken::new();
         let shutdown_embedded_server = || {
             let _ = shutdown_tx.send(());
@@ -599,15 +594,15 @@ impl Drop for RustFSServer {
 /// ```rust,no_run
 /// use rustfs::embedded::{find_available_port, RustFSServerBuilder};
 ///
-/// # async fn example() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-/// let port = find_available_port()?;
-/// let server = RustFSServerBuilder::new()
-///     .address(format!("127.0.0.1:{port}"))
-///     .build()
-///     .await?;
-/// println!("Listening on port {port}");
-/// # Ok(())
-/// # }
+/// async fn example() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+///     let port = find_available_port()?;
+///     let server = RustFSServerBuilder::new()
+///         .address(format!("127.0.0.1:{port}"))
+///         .build()
+///         .await?;
+///     println!("Listening on port {port}");
+///      Ok(())
+///  }
 /// ```
 pub fn find_available_port() -> Result<u16, std::io::Error> {
     let listener = std::net::TcpListener::bind("127.0.0.1:0")?;

rustfs/src/main.rs

@@ -340,14 +340,14 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
     let s3_shutdown_tx = {
         let mut s3_config = config.clone();
         s3_config.console_enable = false;
-        let (s3_shutdown_tx, _) = start_http_server(&s3_config, state_manager.clone(), readiness.clone()).await?;
+        let (s3_shutdown_tx, _) = start_http_server(&s3_config, readiness.clone()).await?;
         Some(s3_shutdown_tx)
     };
 
     let console_shutdown_tx = if config.console_enable && !config.console_address.is_empty() {
         let mut console_config = config.clone();
         console_config.address = console_config.console_address.clone();
-        let (console_shutdown_tx, _) = start_http_server(&console_config, state_manager.clone(), readiness.clone()).await?;
+        let (console_shutdown_tx, _) = start_http_server(&console_config, readiness.clone()).await?;
         Some(console_shutdown_tx)
     } else {
         None
@@ -576,11 +576,11 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
     );
     // 4. Mark as Full Ready now that critical components are warm
     readiness.mark_stage(SystemStage::FullReady);
-    // Update service status to Ready
-    state_manager.update(ServiceState::Ready);
 
     // Set the global RustFS initialization time to now
     rustfs_common::set_global_init_time_now().await;
+    // Publish ready only after all critical bootstrap metadata is in place
+    state_manager.update(ServiceState::Ready);
 
     // Perform hibernation for 1 second
     tokio::time::sleep(SHUTDOWN_TIMEOUT).await;

rustfs/src/server/http.rs

@@ -18,7 +18,7 @@ use crate::auth::IAMAuth;
 use crate::auth_keystone;
 use crate::config;
 use crate::server::{
-    ReadinessGateLayer, RemoteAddr, ServiceState, ServiceStateManager,
+    ReadinessGateLayer, RemoteAddr,
     compress::{CompressionConfig, PathAwareCompressionPredicate, PathCategoryInjectionLayer},
     hybrid::hybrid,
     layer::{
@@ -68,7 +68,6 @@ use tracing_opentelemetry::OpenTelemetrySpanExt;
 
 pub async fn start_http_server(
     config: &config::Config,
-    worker_state_manager: ServiceStateManager,
     readiness: Arc<GlobalReadiness>,
 ) -> Result<(tokio::sync::broadcast::Sender<()>, SocketAddr)> {
     let server_addr = parse_and_resolve_address(config.address.as_str()).map_err(Error::other)?;
@@ -161,12 +160,12 @@ pub async fn start_http_server(
     // Note: outbound material (root CAs, mTLS identity) is already applied in main.rs.
     let tls_snapshot = TlsMaterialSnapshot::load(tls_path)
         .await
-        .map_err(|e| std::io::Error::other(e.to_string()))?;
+        .map_err(|e| Error::other(e.to_string()))?;
 
     let tls_acceptor = tls_snapshot
         .build_tls_acceptor(tls_path)
         .await
-        .map_err(|e| std::io::Error::other(e.to_string()))?;
+        .map_err(|e| Error::other(e.to_string()))?;
     let tls_enabled = tls_acceptor.is_some();
     let protocol = if tls_enabled { "https" } else { "http" };
 
@@ -357,10 +356,6 @@ pub async fn start_http_server(
         let graceful = Arc::new(GracefulShutdown::new());
         debug!("graceful initiated");
 
-        // service ready
-        worker_state_manager.update(ServiceState::Ready);
-        // tls_acceptor is already Option<Arc<TlsAcceptorHolder>>, clone for the loop
-
         loop {
             debug!("Waiting for new connection...");
             let (socket, _) = {
@@ -458,7 +453,6 @@ pub async fn start_http_server(
             process_connection(socket, tls_acceptor.clone(), connection_ctx, graceful.clone());
         }
 
-        worker_state_manager.update(ServiceState::Stopping);
         match Arc::try_unwrap(graceful) {
             Ok(g) => {
                 tokio::select! {
@@ -476,7 +470,6 @@ pub async fn start_http_server(
                 debug!("Timeout reached, forcing shutdown");
             }
         }
-        worker_state_manager.update(ServiceState::Stopped);
     });
 
     Ok((shutdown_tx, local_addr))
@@ -490,7 +483,7 @@ struct ConnectionContext {
     is_console: bool,
     readiness: Arc<GlobalReadiness>,
     /// Pre-computed Keystone auth provider (avoids per-connection OnceLock read).
-    keystone_auth: Option<std::sync::Arc<rustfs_keystone::KeystoneAuthProvider>>,
+    keystone_auth: Option<Arc<rustfs_keystone::KeystoneAuthProvider>>,
     /// Pre-computed trusted proxy layer (avoids per-connection is_enabled() check).
     trusted_proxy_layer: Option<rustfs_trusted_proxies::TrustedProxyLayer>,
 }

rustfs/src/server/service_state.rs

@@ -13,39 +13,18 @@
 // limitations under the License.
 
 use atomic_enum::atomic_enum;
-use std::sync::Arc;
 use std::sync::atomic::Ordering;
+use std::sync::{Arc, Mutex};
 use std::time::Duration;
-use tracing::info;
+use tracing::{info, warn};
 
 // a configurable shutdown timeout
 pub const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(1);
 
-#[cfg(target_os = "linux")]
-fn notify_systemd(state: &str) {
-    use libsystemd::daemon::{NotifyState, notify};
-    use tracing::{debug, error};
-    let notify_state = match state {
-        "ready" => NotifyState::Ready,
-        "stopping" => NotifyState::Stopping,
-        _ => {
-            info!("Unsupported state passed to notify_systemd: {}", state);
-            return;
-        }
-    };
-
-    if let Err(e) = notify(false, &[notify_state]) {
-        error!("Failed to notify systemd: {}", e);
-    } else {
-        debug!("Successfully notified systemd: {}", state);
-    }
-    info!("Systemd notifications are enabled on linux (state: {})", state);
-}
-
-#[cfg(not(target_os = "linux"))]
-fn notify_systemd(state: &str) {
-    info!("Systemd notifications are not available on this platform not linux (state: {})", state);
-}
+const SERVICE_STATUS_STARTING: &str = "Starting";
+const SERVICE_STATUS_RUNNING: &str = "Running";
+const SERVICE_STATUS_STOPPING: &str = "Stopping";
+const SERVICE_STATUS_STOPPED: &str = "Stopped";
 
 #[derive(Debug)]
 pub enum ShutdownSignal {
@@ -100,56 +79,112 @@ pub async fn wait_for_shutdown() -> ShutdownSignal {
 #[derive(Clone)]
 pub struct ServiceStateManager {
     state: Arc<AtomicServiceState>,
+    published_state: Arc<Mutex<Option<ServiceState>>>,
 }
 
 impl ServiceStateManager {
     pub fn new() -> Self {
         Self {
             state: Arc::new(AtomicServiceState::new(ServiceState::Starting)),
+            published_state: Arc::new(Mutex::new(None)),
         }
     }
 
     pub fn update(&self, new_state: ServiceState) {
+        // Serialize transition check + state write + publish dedupe + notify as one
+        // critical section to keep notification order monotonic under concurrency.
+        let mut published_state = self.published_state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+        let current_state = self.current_state();
+        if service_state_rank(new_state) < service_state_rank(current_state) {
+            warn!(
+                current = ?current_state,
+                attempted = ?new_state,
+                "Ignoring regressive service state transition"
+            );
+            return;
+        }
+
         self.state.store(new_state, Ordering::SeqCst);
-        self.notify_systemd(&new_state);
+
+        if *published_state != Some(new_state) {
+            *published_state = Some(new_state);
+            self.notify_systemd(new_state);
+        }
     }
 
     pub fn current_state(&self) -> ServiceState {
         self.state.load(Ordering::SeqCst)
     }
 
-    fn notify_systemd(&self, state: &ServiceState) {
+    fn notify_systemd(&self, state: ServiceState) {
         match state {
             ServiceState::Starting => {
                 info!("RustFS Service is starting...");
-                #[cfg(target_os = "linux")]
-                if let Err(e) =
-                    libsystemd::daemon::notify(false, &[libsystemd::daemon::NotifyState::Status("Starting...".to_string())])
-                {
-                    tracing::error!("Failed to notify systemd of starting state: {}", e);
-                }
+                notify_systemd_daemon(state);
             }
             ServiceState::Ready => {
-                info!("RustFS Service is ready");
-                notify_systemd("ready");
+                info!("RustFS Service is running");
+                notify_systemd_daemon(state);
             }
             ServiceState::Stopping => {
                 info!("RustFS Service is stopping...");
-                notify_systemd("stopping");
+                notify_systemd_daemon(state);
             }
             ServiceState::Stopped => {
                 info!("RustFS Service has stopped");
-                #[cfg(target_os = "linux")]
-                if let Err(e) =
-                    libsystemd::daemon::notify(false, &[libsystemd::daemon::NotifyState::Status("Stopped".to_string())])
-                {
-                    tracing::error!("Failed to notify systemd of stopped state: {}", e);
-                }
+                notify_systemd_daemon(state);
             }
         }
     }
 }
 
+fn service_state_rank(state: ServiceState) -> u8 {
+    match state {
+        ServiceState::Starting => 0,
+        ServiceState::Ready => 1,
+        ServiceState::Stopping => 2,
+        ServiceState::Stopped => 3,
+    }
+}
+
+fn systemd_status_text(state: ServiceState) -> &'static str {
+    match state {
+        ServiceState::Starting => SERVICE_STATUS_STARTING,
+        ServiceState::Ready => SERVICE_STATUS_RUNNING,
+        ServiceState::Stopping => SERVICE_STATUS_STOPPING,
+        ServiceState::Stopped => SERVICE_STATUS_STOPPED,
+    }
+}
+
+#[cfg(target_os = "linux")]
+fn notify_systemd_daemon(state: ServiceState) {
+    use libsystemd::daemon::{NotifyState, notify};
+    use tracing::{debug, error};
+
+    let status = systemd_status_text(state);
+    let result = match state {
+        ServiceState::Starting => notify(false, &[NotifyState::Status(status.to_string())]),
+        ServiceState::Ready => notify(false, &[NotifyState::Ready, NotifyState::Status(status.to_string())]),
+        ServiceState::Stopping => notify(false, &[NotifyState::Stopping, NotifyState::Status(status.to_string())]),
+        ServiceState::Stopped => notify(false, &[NotifyState::Status(status.to_string())]),
+    };
+
+    if let Err(e) = result {
+        error!(%status, ?state, "Failed to notify systemd: {}", e);
+    } else {
+        debug!(%status, ?state, "Successfully notified systemd");
+    }
+}
+
+#[cfg(not(target_os = "linux"))]
+fn notify_systemd_daemon(state: ServiceState) {
+    info!(
+        status = systemd_status_text(state),
+        ?state,
+        "Systemd notifications are not available on this platform"
+    );
+}
+
 impl Default for ServiceStateManager {
     fn default() -> Self {
         Self::new()
@@ -180,4 +215,23 @@ mod tests {
         manager.update(ServiceState::Stopped);
         assert_eq!(manager.current_state(), ServiceState::Stopped);
     }
+
+    #[test]
+    fn test_service_state_manager_ignores_regression() {
+        let manager = ServiceStateManager::new();
+
+        manager.update(ServiceState::Starting);
+        manager.update(ServiceState::Ready);
+        manager.update(ServiceState::Starting);
+        assert_eq!(manager.current_state(), ServiceState::Ready);
+
+        manager.update(ServiceState::Stopping);
+        manager.update(ServiceState::Ready);
+        assert_eq!(manager.current_state(), ServiceState::Stopping);
+    }
+
+    #[test]
+    fn test_ready_maps_to_running_status() {
+        assert_eq!(systemd_status_text(ServiceState::Ready), SERVICE_STATUS_RUNNING);
+    }
 }

rustfs/src/storage/concurrency/io_schedule.rs

@@ -1263,11 +1263,7 @@ impl IoLoadMetrics {
     pub(crate) fn lifetime_average_wait(&self) -> Duration {
         let total = self.total_wait_ns.load(Ordering::Relaxed);
         let count = self.observation_count.load(Ordering::Relaxed);
-        if count == 0 {
-            Duration::ZERO
-        } else {
-            Duration::from_nanos(total / count)
-        }
+        total.checked_div(count).map(Duration::from_nanos).unwrap_or(Duration::ZERO)
     }
 
     /// Get the total observation count

rustfs/src/storage/lock_optimizer.rs

@@ -147,11 +147,7 @@ impl LockStats {
     pub fn avg_hold_time(&self) -> Duration {
         let total = self.total_hold_time_us.load(Ordering::Relaxed);
         let count = self.locks_released_early.load(Ordering::Relaxed);
-        if count > 0 {
-            Duration::from_micros(total / count)
-        } else {
-            Duration::ZERO
-        }
+        total.checked_div(count).map(Duration::from_micros).unwrap_or(Duration::ZERO)
     }
 
     /// Get maximum hold time.