fix(targets): unify runtime health snapshots (#5110)

* fix(targets): unify runtime health snapshots

* fix(targets): stabilize health snapshot merge

* fix(admin): import runtime health test type

rustfs/src/admin/handlers/audit.rs

@@ -16,8 +16,8 @@ use crate::admin::{
     auth::validate_admin_request,
     handlers::audit_runtime_config::{load_server_config_from_store, update_audit_config_and_reload},
     handlers::target_descriptor::{
-        AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, build_enabled_target_kvs,
-        build_json_response, collect_runtime_statuses, extract_supported_target_params,
+        AdminTargetSpec, EndpointKey, RuntimeHealthStatus, TargetEndpointSource, admin_target_spec_from_builtin,
+        build_enabled_target_kvs, build_json_response, collect_runtime_statuses, extract_supported_target_params,
         merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
         target_mutation_block_reason as shared_target_mutation_block_reason,
     },
@@ -192,6 +192,8 @@ struct AuditEndpoint {
     account_id: String,
     service: String,
     status: String,
+    health_state: String,
+    health_reason: String,
     source: TargetEndpointSource,
 }
 
@@ -248,7 +250,10 @@ async fn audit_target_operation_block_reason(action: &str) -> Option<String> {
     target_module_disabled_reason("audit", rustfs_config::ENV_AUDIT_ENABLE, is_audit_module_enabled(), action)
 }
 
-fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> S3Result<Vec<AuditEndpoint>> {
+fn merge_audit_endpoints(
+    config: &Config,
+    runtime_statuses: HashMap<EndpointKey, RuntimeHealthStatus>,
+) -> S3Result<Vec<AuditEndpoint>> {
     Ok(
         shared_merge_target_endpoints(audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses)?
             .into_iter()
@@ -256,6 +261,8 @@ fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey,
                 account_id: endpoint.account_id,
                 service: endpoint.service,
                 status: endpoint.status,
+                health_state: endpoint.health_state,
+                health_reason: endpoint.health_reason,
                 source: endpoint.source,
             })
             .collect(),
@@ -428,6 +435,14 @@ mod tests {
         }])
     }
 
+    fn online_health() -> RuntimeHealthStatus {
+        RuntimeHealthStatus {
+            status: "online".to_string(),
+            state: "online".to_string(),
+            reason: "reachable".to_string(),
+        }
+    }
+
     fn with_audit_webhook_target_env_cleared<F>(target_name: &str, f: F)
     where
         F: FnOnce(),
@@ -472,8 +487,8 @@ mod tests {
             ],
             || {
                 let runtime = HashMap::from([
-                    (("mixed-target".to_string(), "webhook".to_string()), "online".to_string()),
-                    (("env-only".to_string(), "webhook".to_string()), "online".to_string()),
+                    (("mixed-target".to_string(), "webhook".to_string()), online_health()),
+                    (("env-only".to_string(), "webhook".to_string()), online_health()),
                 ]);
                 let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");
 
@@ -515,8 +530,8 @@ mod tests {
             ],
             || {
                 let runtime = HashMap::from([
-                    (("mixed-kafka".to_string(), "kafka".to_string()), "online".to_string()),
-                    (("env-kafka".to_string(), "kafka".to_string()), "online".to_string()),
+                    (("mixed-kafka".to_string(), "kafka".to_string()), online_health()),
+                    (("env-kafka".to_string(), "kafka".to_string()), online_health()),
                 ]);
                 let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");
 
@@ -552,8 +567,8 @@ mod tests {
             ],
             || {
                 let runtime = HashMap::from([
-                    (("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()),
-                    (("env-amqp".to_string(), "amqp".to_string()), "online".to_string()),
+                    (("mixed-amqp".to_string(), "amqp".to_string()), online_health()),
+                    (("env-amqp".to_string(), "amqp".to_string()), online_health()),
                 ]);
                 let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");
 
@@ -775,7 +790,7 @@ mod tests {
                 ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARYCASE", Some("https://example.com/hook")),
             ],
             || {
-                let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]);
+                let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), online_health())]);
                 let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");
                 let mixed = merged
                     .iter()

rustfs/src/admin/handlers/event.rs

@@ -17,8 +17,8 @@ use crate::admin::{
     handlers::notify_runtime_access::{get_notification_system, load_notification_config_snapshot},
     handlers::supervise_admin_mutation,
     handlers::target_descriptor::{
-        AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, build_enabled_target_kvs,
-        build_json_response, collect_runtime_statuses, extract_supported_target_params,
+        AdminTargetSpec, EndpointKey, RuntimeHealthStatus, TargetEndpointSource, admin_target_spec_from_builtin,
+        build_enabled_target_kvs, build_json_response, collect_runtime_statuses, extract_supported_target_params,
         merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
         target_mutation_block_reason as shared_target_mutation_block_reason,
     },
@@ -240,6 +240,8 @@ struct NotificationEndpoint {
     account_id: String,
     service: String,
     status: String,
+    health_state: String,
+    health_reason: String,
     source: TargetEndpointSource,
 }
 
@@ -301,7 +303,7 @@ async fn notification_target_operation_block_reason(action: &str) -> Option<Stri
 
 fn merge_notification_endpoints(
     config: &Config,
-    runtime_statuses: HashMap<EndpointKey, String>,
+    runtime_statuses: HashMap<EndpointKey, RuntimeHealthStatus>,
 ) -> S3Result<Vec<NotificationEndpoint>> {
     Ok(
         shared_merge_target_endpoints(notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses)?
@@ -310,6 +312,8 @@ fn merge_notification_endpoints(
                 account_id: endpoint.account_id,
                 service: endpoint.service,
                 status: endpoint.status,
+                health_state: endpoint.health_state,
+                health_reason: endpoint.health_reason,
                 source: endpoint.source,
             })
             .collect(),
@@ -463,7 +467,7 @@ impl Operation for ListTargetsArns {
         let target_statuses = collect_runtime_statuses(ns.get_target_values().await)
             .await
             .into_iter()
-            .map(|((account_id, service), status)| (rustfs_targets::arn::TargetID::new(account_id, service), status))
+            .map(|((account_id, service), health)| (rustfs_targets::arn::TargetID::new(account_id, service), health.status))
             .collect();
 
         let data_target_arn_list = collect_online_target_arns(region.as_str(), target_statuses);
@@ -548,6 +552,15 @@ mod tests {
         }])
     }
 
+    fn runtime_health(status: &str) -> RuntimeHealthStatus {
+        let online = status == "online";
+        RuntimeHealthStatus {
+            status: status.to_string(),
+            state: if online { "online" } else { "offline" }.to_string(),
+            reason: if online { "reachable" } else { "unreachable" }.to_string(),
+        }
+    }
+
     #[test]
     fn notification_target_subsystem_resolves_admin_route_type() {
         assert_eq!(
@@ -565,6 +578,8 @@ mod tests {
                 account_id: "primary".to_string(),
                 service: "webhook".to_string(),
                 status: "online".to_string(),
+                health_state: "online".to_string(),
+                health_reason: "reachable".to_string(),
                 source: TargetEndpointSource::Config,
             }],
         };
@@ -577,6 +592,8 @@ mod tests {
                     "account_id": "primary",
                     "service": "webhook",
                     "status": "online",
+                    "health_state": "online",
+                    "health_reason": "reachable",
                     "source": "config"
                 }]
             })
@@ -596,7 +613,14 @@ mod tests {
         );
         let config = Config(cfg_map);
 
-        let runtime = HashMap::from([(("webhook-a".to_string(), "webhook".to_string()), "online".to_string())]);
+        let runtime = HashMap::from([(
+            ("webhook-a".to_string(), "webhook".to_string()),
+            RuntimeHealthStatus {
+                status: "offline".to_string(),
+                state: "error".to_string(),
+                reason: "timed_out".to_string(),
+            },
+        )]);
         let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
 
         let mqtt = merged
@@ -610,7 +634,9 @@ mod tests {
             .iter()
             .find(|entry| entry.account_id == "webhook-a" && entry.service == "webhook")
             .expect("webhook-a should be present");
-        assert_eq!(webhook.status, "online");
+        assert_eq!(webhook.status, "offline");
+        assert_eq!(webhook.health_state, "error");
+        assert_eq!(webhook.health_reason, "timed_out");
         assert_eq!(webhook.source, TargetEndpointSource::Config);
     }
 
@@ -623,8 +649,8 @@ mod tests {
         let config = Config(HashMap::from([(NOTIFY_WEBHOOK_SUB_SYS.to_string(), webhook_targets)]));
 
         let runtime = HashMap::from([
-            (("webhook-enabled".to_string(), "webhook".to_string()), "online".to_string()),
-            (("env-only".to_string(), "mqtt".to_string()), "offline".to_string()),
+            (("webhook-enabled".to_string(), "webhook".to_string()), runtime_health("online")),
+            (("env-only".to_string(), "mqtt".to_string()), runtime_health("offline")),
         ]);
         let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
 
@@ -665,8 +691,8 @@ mod tests {
             ],
             || {
                 let runtime = HashMap::from([
-                    (("mixed-target".to_string(), "webhook".to_string()), "online".to_string()),
-                    (("env-only".to_string(), "webhook".to_string()), "online".to_string()),
+                    (("mixed-target".to_string(), "webhook".to_string()), runtime_health("online")),
+                    (("env-only".to_string(), "webhook".to_string()), runtime_health("online")),
                 ]);
                 let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
 
@@ -708,8 +734,8 @@ mod tests {
             ],
             || {
                 let runtime = HashMap::from([
-                    (("mixed-kafka".to_string(), "kafka".to_string()), "online".to_string()),
-                    (("env-kafka".to_string(), "kafka".to_string()), "online".to_string()),
+                    (("mixed-kafka".to_string(), "kafka".to_string()), runtime_health("online")),
+                    (("env-kafka".to_string(), "kafka".to_string()), runtime_health("online")),
                 ]);
                 let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
 
@@ -745,8 +771,8 @@ mod tests {
             ],
             || {
                 let runtime = HashMap::from([
-                    (("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()),
-                    (("env-amqp".to_string(), "amqp".to_string()), "online".to_string()),
+                    (("mixed-amqp".to_string(), "amqp".to_string()), runtime_health("online")),
+                    (("env-amqp".to_string(), "amqp".to_string()), runtime_health("online")),
                 ]);
                 let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
 
@@ -922,7 +948,7 @@ mod tests {
                 ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARYCASE", Some("https://example.com/hook")),
             ],
             || {
-                let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]);
+                let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), runtime_health("online"))]);
                 let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
                 let mixed = merged
                     .iter()

rustfs/src/admin/handlers/extensions.rs

@@ -486,6 +486,8 @@ mod tests {
             account_id: "primary".to_string(),
             service: "webhook".to_string(),
             status: "offline".to_string(),
+            health_state: "offline".to_string(),
+            health_reason: "not_loaded_in_runtime".to_string(),
             source: PluginInstanceSource::Config,
             enabled: true,
             config: HashMap::from([("endpoint".to_string(), "https://example.test/webhook".to_string())]),

rustfs/src/admin/handlers/plugins_instances.rs

@@ -19,8 +19,8 @@ use crate::admin::{
         load_notification_config_snapshot, remove_notification_target_config, set_notification_target_config,
     },
     handlers::target_descriptor::{
-        AdminTargetSpec, TargetEndpointSource, TargetInstanceReadModel, admin_target_spec_from_builtin, build_enabled_target_kvs,
-        build_json_response, collect_runtime_statuses, collect_target_instances, find_target_instance,
+        AdminTargetSpec, RuntimeHealthStatus, TargetEndpointSource, TargetInstanceReadModel, admin_target_spec_from_builtin,
+        build_enabled_target_kvs, build_json_response, collect_runtime_statuses, collect_target_instances, find_target_instance,
         target_module_disabled_reason, target_mutation_block_reason as shared_target_mutation_block_reason,
     },
     plugin_contract::{
@@ -205,6 +205,8 @@ fn map_instance(instance: TargetInstanceReadModel) -> PluginInstanceEntry {
         account_id: instance.account_id,
         service: instance.service,
         status: instance.status,
+        health_state: instance.health_state,
+        health_reason: instance.health_reason,
         source: map_instance_source(instance.source),
         enabled: instance.enabled,
         config,
@@ -639,7 +641,9 @@ async fn plugin_instance_operation_block_reason(context: PluginInstanceDomainCon
     module_disabled_block_reason(context.domain, action)
 }
 
-async fn plugin_instance_runtime_statuses(context: PluginInstanceDomainContext) -> S3Result<HashMap<(String, String), String>> {
+async fn plugin_instance_runtime_statuses(
+    context: PluginInstanceDomainContext,
+) -> S3Result<HashMap<(String, String), RuntimeHealthStatus>> {
     match context.domain {
         PluginContractDomain::Notify => {
             let (ns, _) = load_notification_config_snapshot().await?;
@@ -865,7 +869,8 @@ mod tests {
         parse_plugin_instance_id, parse_plugin_instance_source, resolve_plugin_instance_target,
     };
     use crate::admin::handlers::target_descriptor::{
-        TargetEndpointSource, TargetInstanceReadModel, canonical_target_instance_id, collect_target_instances,
+        RuntimeHealthStatus, TargetEndpointSource, TargetInstanceReadModel, canonical_target_instance_id,
+        collect_target_instances,
     };
     use crate::admin::plugin_contract::{
         PluginInstanceDiagnosticCode, PluginInstanceDiagnosticCount, PluginInstanceEntry, PluginInstanceSource,
@@ -967,9 +972,46 @@ mod tests {
             .expect("configured instance should be present");
 
         assert_eq!(primary.status, "offline");
+        assert_eq!(primary.health_state, "offline");
+        assert_eq!(primary.health_reason, "not_loaded_in_runtime");
         assert_eq!(primary.source, TargetEndpointSource::Config);
     }
 
+    #[test]
+    fn disabled_instance_without_runtime_reports_disabled_health() {
+        let config = Config(HashMap::from([(
+            NOTIFY_WEBHOOK_SUB_SYS.to_string(),
+            HashMap::from([(
+                "primary".to_string(),
+                KVS(vec![
+                    KV {
+                        key: ENABLE_KEY.to_string(),
+                        value: "off".to_string(),
+                        hidden_if_empty: false,
+                    },
+                    KV {
+                        key: WEBHOOK_ENDPOINT.to_string(),
+                        value: "https://example.com/webhook".to_string(),
+                        hidden_if_empty: false,
+                    },
+                ]),
+            )]),
+        )]));
+
+        let instances =
+            collect_target_instances(super::notification_target_specs(), NOTIFY_ROUTE_PREFIX, &config, HashMap::new())
+                .expect("collect target instances");
+        let primary = instances
+            .into_iter()
+            .find(|instance| instance.account_id == "primary" && instance.service == "webhook")
+            .expect("disabled instance should be present");
+
+        assert_eq!(primary.status, "offline");
+        assert_eq!(primary.health_state, "disabled");
+        assert_eq!(primary.health_reason, "disabled");
+        assert!(!primary.runtime_present);
+    }
+
     #[test]
     fn env_only_instance_appears_with_env_source() {
         temp_env::with_vars(
@@ -998,7 +1040,14 @@ mod tests {
 
     #[test]
     fn runtime_only_instance_appears_with_runtime_source() {
-        let runtime_statuses = HashMap::from([(("runtime-only".to_string(), "webhook".to_string()), "online".to_string())]);
+        let runtime_statuses = HashMap::from([(
+            ("runtime-only".to_string(), "webhook".to_string()),
+            RuntimeHealthStatus {
+                status: "online".to_string(),
+                state: "online".to_string(),
+                reason: "reachable".to_string(),
+            },
+        )]);
         let instances = collect_target_instances(
             super::notification_target_specs(),
             NOTIFY_ROUTE_PREFIX,
@@ -1014,6 +1063,8 @@ mod tests {
 
         assert_eq!(runtime_only.source, TargetEndpointSource::Runtime);
         assert_eq!(runtime_only.status, "online");
+        assert_eq!(runtime_only.health_state, "online");
+        assert_eq!(runtime_only.health_reason, "reachable");
         assert_eq!(runtime_only.plugin_id, "builtin:webhook");
         assert_eq!(runtime_only.subsystem, NOTIFY_WEBHOOK_SUB_SYS);
     }
@@ -1028,6 +1079,8 @@ mod tests {
             account_id: "Primary".to_string(),
             service: "webhook".to_string(),
             status: "offline".to_string(),
+            health_state: "offline".to_string(),
+            health_reason: "not_loaded_in_runtime".to_string(),
             runtime_present: false,
             source: TargetEndpointSource::Config,
             enabled: true,
@@ -1050,6 +1103,8 @@ mod tests {
             account_id: "primary".to_string(),
             service: "webhook".to_string(),
             status: "online".to_string(),
+            health_state: "online".to_string(),
+            health_reason: "reachable".to_string(),
             runtime_present: true,
             source: TargetEndpointSource::Config,
             enabled: true,
@@ -1481,6 +1536,8 @@ mod tests {
             account_id: "primary".to_string(),
             service: "webhook".to_string(),
             status: "offline".to_string(),
+            health_state: "offline".to_string(),
+            health_reason: "not_loaded_in_runtime".to_string(),
             runtime_present: false,
             source: TargetEndpointSource::Config,
             enabled: true,
@@ -1505,6 +1562,8 @@ mod tests {
             account_id: "primary".to_string(),
             service: "webhook".to_string(),
             status: "offline".to_string(),
+            health_state: "offline".to_string(),
+            health_reason: "unreachable".to_string(),
             runtime_present: true,
             source: TargetEndpointSource::Runtime,
             enabled: true,
@@ -1529,6 +1588,8 @@ mod tests {
             account_id: "primary".to_string(),
             service: "webhook".to_string(),
             status: "offline".to_string(),
+            health_state: "disabled".to_string(),
+            health_reason: "disabled".to_string(),
             runtime_present: false,
             source: TargetEndpointSource::Mixed,
             enabled: false,
@@ -1570,6 +1631,8 @@ mod tests {
             account_id: "primary".to_string(),
             service: "webhook".to_string(),
             status: "offline".to_string(),
+            health_state: "offline".to_string(),
+            health_reason: "not_loaded_in_runtime".to_string(),
             runtime_present: false,
             source: TargetEndpointSource::Config,
             enabled: true,
@@ -1683,6 +1746,13 @@ mod tests {
             account_id: input.account_id.to_string(),
             service: input.service.to_string(),
             status: input.status.to_string(),
+            health_state: input.status.to_string(),
+            health_reason: if input.status == "online" {
+                "reachable"
+            } else {
+                "unreachable"
+            }
+            .to_string(),
             source: input.source,
             enabled: input.enabled,
             config: HashMap::new(),

rustfs/src/admin/handlers/target_descriptor.rs

@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use futures::StreamExt;
 use futures::future::BoxFuture;
 use hashbrown::HashSet as HbHashSet;
 use http::{HeaderMap, HeaderValue, StatusCode};
@@ -22,12 +21,11 @@ use rustfs_config::{
     MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC,
     MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_QUEUE_DIR, POSTGRES_QUEUE_DIR, REDIS_QUEUE_DIR,
 };
-use rustfs_targets::SharedTarget;
 use rustfs_targets::{
-    BuiltinTargetAdminDescriptor, TargetAdminMetadata, TargetDomain, TargetError, TargetRequestValidator,
-    check_amqp_broker_available, check_kafka_broker_available, check_mqtt_broker_available_with_tls,
-    check_mysql_server_available, check_nats_server_available, check_postgres_server_available, check_pulsar_broker_available,
-    check_redis_server_available,
+    BuiltinTargetAdminDescriptor, SharedTarget, TargetAdminMetadata, TargetDomain, TargetError, TargetHealthReason,
+    TargetHealthState, TargetRequestValidator, check_amqp_broker_available, check_kafka_broker_available,
+    check_mqtt_broker_available_with_tls, check_mysql_server_available, check_nats_server_available,
+    check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available,
     config::{
         TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, build_amqp_args, build_kafka_args, build_mysql_args,
         build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args, try_normalize_target_plugin_instances,
@@ -42,11 +40,36 @@ use std::collections::{HashMap, HashSet};
 use std::io::{Error, ErrorKind};
 use std::path::Path;
 use std::sync::Arc;
-use tokio::sync::Semaphore;
 use tokio::time::{Duration, sleep, timeout};
 use url::Url;
 
 pub(crate) type EndpointKey = (String, String);
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(crate) struct RuntimeHealthStatus {
+    pub status: String,
+    pub state: String,
+    pub reason: String,
+}
+
+impl RuntimeHealthStatus {
+    fn disabled() -> Self {
+        Self {
+            status: TargetHealthState::Disabled.status().to_string(),
+            state: TargetHealthState::Disabled.as_str().to_string(),
+            reason: TargetHealthReason::Disabled.as_str().to_string(),
+        }
+    }
+
+    fn not_loaded() -> Self {
+        Self {
+            status: TargetHealthState::Offline.status().to_string(),
+            state: TargetHealthState::Offline.as_str().to_string(),
+            reason: TargetHealthReason::NotLoadedInRuntime.as_str().to_string(),
+        }
+    }
+}
+
 type AdminRequestValidatorFn =
     Arc<dyn for<'a> Fn(&'a HashMap<String, String>, &'a str) -> BoxFuture<'a, S3Result<()>> + Send + Sync>;
 type DomainScopedValidatorFn = for<'a> fn(&'a HashMap<String, String>, &'a str, TargetDomain) -> BoxFuture<'a, S3Result<()>>;
@@ -64,6 +87,8 @@ pub(crate) struct MergedTargetEndpoint {
     pub account_id: String,
     pub service: String,
     pub status: String,
+    pub health_state: String,
+    pub health_reason: String,
     pub source: TargetEndpointSource,
 }
 
@@ -76,6 +101,8 @@ pub(crate) struct TargetInstanceReadModel {
     pub account_id: String,
     pub service: String,
     pub status: String,
+    pub health_state: String,
+    pub health_reason: String,
     pub runtime_present: bool,
     pub source: TargetEndpointSource,
     pub enabled: bool,
@@ -272,43 +299,36 @@ pub(crate) fn build_json_response(
     S3Response::with_headers((status, body), header)
 }
 
-pub(crate) async fn collect_runtime_statuses<E>(targets: Vec<SharedTarget<E>>) -> HashMap<EndpointKey, String>
+pub(crate) async fn collect_runtime_statuses<E>(targets: Vec<SharedTarget<E>>) -> HashMap<EndpointKey, RuntimeHealthStatus>
 where
-    E: Send + Sync + 'static + Clone + serde::Serialize + serde::de::DeserializeOwned,
+    E: rustfs_targets::PluginEvent,
 {
-    let semaphore = Arc::new(Semaphore::new(10));
-    let mut futures = futures::stream::FuturesUnordered::new();
-
-    for target in targets {
-        let sem = Arc::clone(&semaphore);
-        futures.push(async move {
-            let _permit = sem.acquire().await;
-            let status = match tokio::time::timeout(Duration::from_secs(3), target.is_active()).await {
-                Ok(Ok(true)) => "online",
-                _ => "offline",
-            };
-            ((target.id().id, target.id().name), status.to_string())
-        });
-    }
-
-    let mut runtime_statuses = HashMap::new();
-    while let Some((key, status)) = futures.next().await {
-        runtime_statuses.insert(key, status);
-    }
-
-    runtime_statuses
+    rustfs_targets::health_snapshots_for_targets(targets)
+        .await
+        .into_iter()
+        .map(|snapshot| {
+            (
+                (snapshot.account_id, snapshot.target_type),
+                RuntimeHealthStatus {
+                    status: snapshot.state.status().to_string(),
+                    state: snapshot.state.as_str().to_string(),
+                    reason: snapshot.reason.as_str().to_string(),
+                },
+            )
+        })
+        .collect()
 }
 
 pub(crate) fn merge_target_endpoints(
     specs: &[AdminTargetSpec],
     route_prefix: &str,
     config: &Config,
-    runtime_statuses: HashMap<EndpointKey, String>,
+    runtime_statuses: HashMap<EndpointKey, RuntimeHealthStatus>,
 ) -> S3Result<Vec<MergedTargetEndpoint>> {
     let mut endpoints = Vec::new();
     let mut seen = HashSet::new();
     let snapshot = collect_endpoint_snapshot(specs, route_prefix, config)?;
-    let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
+    let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, RuntimeHealthStatus)> = HashMap::new();
 
     for ((account_id, service), status) in runtime_statuses {
         let normalized = normalized_endpoint_key(&account_id, &service);
@@ -323,15 +343,17 @@ pub(crate) fn merge_target_endpoints(
             continue;
         }
 
-        let status = normalized_runtime_statuses
+        let health = normalized_runtime_statuses
             .remove(&normalized)
             .map(|(_, _, status)| status)
-            .unwrap_or_else(|| "offline".to_string());
+            .unwrap_or_else(RuntimeHealthStatus::not_loaded);
 
         endpoints.push(MergedTargetEndpoint {
             account_id: key.0,
             service: key.1,
-            status,
+            status: health.status,
+            health_state: health.state,
+            health_reason: health.reason,
             source: classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &normalized),
         });
     }
@@ -341,7 +363,9 @@ pub(crate) fn merge_target_endpoints(
             endpoints.push(MergedTargetEndpoint {
                 account_id,
                 service,
-                status,
+                status: status.status,
+                health_state: status.state,
+                health_reason: status.reason,
                 source: classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &normalized),
             });
         }
@@ -356,6 +380,8 @@ pub(crate) fn merge_target_endpoints(
             account_id: key.0.clone(),
             service: key.1.clone(),
             status: "offline".to_string(),
+            health_state: TargetHealthState::Offline.as_str().to_string(),
+            health_reason: TargetHealthReason::NotLoadedInRuntime.as_str().to_string(),
             source: classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, key),
         });
     }
@@ -372,11 +398,11 @@ pub(crate) fn collect_target_instances(
     specs: &[AdminTargetSpec],
     route_prefix: &str,
     config: &Config,
-    runtime_statuses: HashMap<EndpointKey, String>,
+    runtime_statuses: HashMap<EndpointKey, RuntimeHealthStatus>,
 ) -> S3Result<Vec<TargetInstanceReadModel>> {
     let mut instances = Vec::new();
     let mut seen = HashSet::new();
-    let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
+    let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, RuntimeHealthStatus)> = HashMap::new();
     let domain = inferred_target_domain(route_prefix);
     let snapshot = collect_endpoint_snapshot(specs, route_prefix, config)?;
 
@@ -394,10 +420,16 @@ pub(crate) fn collect_target_instances(
         }
 
         let runtime_present = normalized_runtime_statuses.contains_key(&key);
-        let status = normalized_runtime_statuses
+        let health = normalized_runtime_statuses
             .remove(&key)
             .map(|(_, _, status)| status)
-            .unwrap_or_else(|| "offline".to_string());
+            .unwrap_or_else(|| {
+                if instance.enabled {
+                    RuntimeHealthStatus::not_loaded()
+                } else {
+                    RuntimeHealthStatus::disabled()
+                }
+            });
         let source = classify_endpoint_source_flags(instance_has_config_entry(&instance), instance_has_env_entry(&instance));
 
         instances.push(TargetInstanceReadModel {
@@ -407,7 +439,9 @@ pub(crate) fn collect_target_instances(
             subsystem: instance.subsystem,
             account_id: instance.instance_id,
             service: instance.target_type,
-            status,
+            status: health.status,
+            health_state: health.state,
+            health_reason: health.reason,
             runtime_present,
             source,
             enabled: instance.enabled,
@@ -430,7 +464,9 @@ pub(crate) fn collect_target_instances(
             subsystem,
             account_id,
             service,
-            status,
+            status: status.status,
+            health_state: status.state,
+            health_reason: status.reason,
             runtime_present: true,
             source: TargetEndpointSource::Runtime,
             enabled: true,
@@ -446,7 +482,7 @@ pub(crate) fn find_target_instance(
     specs: &[AdminTargetSpec],
     route_prefix: &str,
     config: &Config,
-    runtime_statuses: HashMap<EndpointKey, String>,
+    runtime_statuses: HashMap<EndpointKey, RuntimeHealthStatus>,
     canonical_id: &str,
 ) -> S3Result<Option<TargetInstanceReadModel>> {
     Ok(collect_target_instances(specs, route_prefix, config, runtime_statuses)?

rustfs/src/admin/plugin_contract.rs

@@ -323,6 +323,8 @@ pub(crate) struct PluginInstanceEntry {
     pub account_id: String,
     pub service: String,
     pub status: String,
+    pub health_state: String,
+    pub health_reason: String,
     pub source: PluginInstanceSource,
     pub enabled: bool,
     pub config: HashMap<String, String>,
@@ -482,6 +484,8 @@ mod tests {
                 account_id: "primary".to_string(),
                 service: "webhook".to_string(),
                 status: "offline".to_string(),
+                health_state: "offline".to_string(),
+                health_reason: "not_loaded_in_runtime".to_string(),
                 source: PluginInstanceSource::Config,
                 enabled: true,
                 config: HashMap::from([
@@ -512,6 +516,8 @@ mod tests {
                     "account_id": "primary",
                     "service": "webhook",
                     "status": "offline",
+                    "health_state": "offline",
+                    "health_reason": "not_loaded_in_runtime",
                     "source": "config",
                     "enabled": true,
                     "config": {
@@ -541,6 +547,8 @@ mod tests {
                 account_id: "primary".to_string(),
                 service: "webhook".to_string(),
                 status: "offline".to_string(),
+                health_state: "offline".to_string(),
+                health_reason: "not_loaded_in_runtime".to_string(),
                 source: PluginInstanceSource::Config,
                 enabled: true,
                 config: HashMap::from([("endpoint".to_string(), "https://example.com/hook".to_string())]),
@@ -565,6 +573,8 @@ mod tests {
                 "account_id": "primary",
                 "service": "webhook",
                 "status": "offline",
+                "health_state": "offline",
+                "health_reason": "not_loaded_in_runtime",
                 "source": "config",
                 "enabled": true,
                 "config": {

rustfs/src/admin/snapshots/rustfs__admin__plugin_contract__tests__plugin_instance_contract.snap

@@ -21,6 +21,8 @@ expression: value
       ],
       "domain": "notify",
       "enabled": true,
+      "health_reason": "not_loaded_in_runtime",
+      "health_state": "offline",
       "id": "builtin:webhook:notify:primary",
       "plugin_id": "builtin:webhook",
       "service": "webhook",

rustfs/src/admin/snapshots/rustfs__admin__plugin_contract__tests__plugin_instance_detail_contract.snap

@@ -18,6 +18,8 @@ expression: value
   ],
   "domain": "notify",
   "enabled": true,
+  "health_reason": "not_loaded_in_runtime",
+  "health_state": "offline",
   "id": "builtin:webhook:notify:primary",
   "plugin_id": "builtin:webhook",
   "service": "webhook",