mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-30 16:59:52 +00:00
Improve health check handlers for endpoint and console (GET/HEAD, safer error handling) (#942)
* Improve health check handlers for endpoint and console - Add unified GET/HEAD handling for `/health` and `/rustfs/console/health` - Implement proper method filtering and 405 with `Allow: GET, HEAD` - Avoid panics by removing `unwrap()` in health check logic - Add safe fallbacks for JSON serialization and uptime calculation - Ensure HEAD requests return only status and headers (empty body) - Keep response format backward compatible for monitoring systems * fix
This commit is contained in:
+95
-33
@@ -15,7 +15,7 @@
|
|||||||
use crate::config::build;
|
use crate::config::build;
|
||||||
use crate::license::get_license;
|
use crate::license::get_license;
|
||||||
use axum::{
|
use axum::{
|
||||||
Json, Router,
|
Router,
|
||||||
body::Body,
|
body::Body,
|
||||||
extract::Request,
|
extract::Request,
|
||||||
middleware,
|
middleware,
|
||||||
@@ -405,7 +405,7 @@ fn setup_console_middleware_stack(
|
|||||||
.route("/favicon.ico", get(static_handler))
|
.route("/favicon.ico", get(static_handler))
|
||||||
.route(&format!("{CONSOLE_PREFIX}/license"), get(license_handler))
|
.route(&format!("{CONSOLE_PREFIX}/license"), get(license_handler))
|
||||||
.route(&format!("{CONSOLE_PREFIX}/config.json"), get(config_handler))
|
.route(&format!("{CONSOLE_PREFIX}/config.json"), get(config_handler))
|
||||||
.route(&format!("{CONSOLE_PREFIX}/health"), get(health_check))
|
.route(&format!("{CONSOLE_PREFIX}/health"), get(health_check).head(health_check))
|
||||||
.nest(CONSOLE_PREFIX, Router::new().fallback_service(get(static_handler)))
|
.nest(CONSOLE_PREFIX, Router::new().fallback_service(get(static_handler)))
|
||||||
.fallback_service(get(static_handler));
|
.fallback_service(get(static_handler));
|
||||||
|
|
||||||
@@ -437,42 +437,104 @@ fn setup_console_middleware_stack(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Console health check handler with comprehensive health information
|
/// Console health check handler with comprehensive health information
|
||||||
async fn health_check() -> Json<serde_json::Value> {
|
async fn health_check(method: Method) -> Response {
|
||||||
use rustfs_ecstore::new_object_layer_fn;
|
let builder = Response::builder()
|
||||||
|
.status(StatusCode::OK)
|
||||||
|
.header("content-type", "application/json");
|
||||||
|
match method {
|
||||||
|
// GET: Returns complete JSON
|
||||||
|
Method::GET => {
|
||||||
|
let mut health_status = "ok";
|
||||||
|
let mut details = json!({});
|
||||||
|
|
||||||
let mut health_status = "ok";
|
// Check storage backend health
|
||||||
let mut details = json!({});
|
if let Some(_store) = rustfs_ecstore::new_object_layer_fn() {
|
||||||
|
details["storage"] = json!({"status": "connected"});
|
||||||
|
} else {
|
||||||
|
health_status = "degraded";
|
||||||
|
details["storage"] = json!({"status": "disconnected"});
|
||||||
|
}
|
||||||
|
|
||||||
// Check storage backend health
|
// Check IAM system health
|
||||||
if let Some(_store) = new_object_layer_fn() {
|
match rustfs_iam::get() {
|
||||||
details["storage"] = json!({"status": "connected"});
|
Ok(_) => {
|
||||||
} else {
|
details["iam"] = json!({"status": "connected"});
|
||||||
health_status = "degraded";
|
}
|
||||||
details["storage"] = json!({"status": "disconnected"});
|
Err(_) => {
|
||||||
}
|
health_status = "degraded";
|
||||||
|
details["iam"] = json!({"status": "disconnected"});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Check IAM system health
|
let body_json = json!({
|
||||||
match rustfs_iam::get() {
|
"status": health_status,
|
||||||
Ok(_) => {
|
"service": "rustfs-console",
|
||||||
details["iam"] = json!({"status": "connected"});
|
"timestamp": chrono::Utc::now().to_rfc3339(),
|
||||||
|
"version": env!("CARGO_PKG_VERSION"),
|
||||||
|
"details": details,
|
||||||
|
"uptime": std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs()
|
||||||
|
});
|
||||||
|
|
||||||
|
// Return a minimal JSON when serialization fails to avoid panic
|
||||||
|
let body_str = serde_json::to_string(&body_json).unwrap_or_else(|e| {
|
||||||
|
error!(
|
||||||
|
target: "rustfs::console::health",
|
||||||
|
"failed to serialize health check body: {}",
|
||||||
|
e
|
||||||
|
);
|
||||||
|
// Simplified back-up JSON
|
||||||
|
"{\"status\":\"error\",\"service\":\"rustfs-console\"}".to_string()
|
||||||
|
});
|
||||||
|
builder.body(Body::from(body_str)).unwrap_or_else(|e| {
|
||||||
|
error!(
|
||||||
|
target: "rustfs::console::health",
|
||||||
|
"failed to build GET health response: {}",
|
||||||
|
e
|
||||||
|
);
|
||||||
|
Response::builder()
|
||||||
|
.status(StatusCode::INTERNAL_SERVER_ERROR)
|
||||||
|
.body(Body::from("failed to build response"))
|
||||||
|
.unwrap_or_else(|_| Response::new(Body::from("")))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
Err(_) => {
|
|
||||||
health_status = "degraded";
|
|
||||||
details["iam"] = json!({"status": "disconnected"});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Json(json!({
|
// HEAD: Only status + headers are returned, body is empty
|
||||||
"status": health_status,
|
Method::HEAD => builder.body(Body::empty()).unwrap_or_else(|e| {
|
||||||
"service": "rustfs-console",
|
error!(
|
||||||
"timestamp": chrono::Utc::now().to_rfc3339(),
|
target: "rustfs::console::health",
|
||||||
"version": env!("CARGO_PKG_VERSION"),
|
"failed to build HEAD health response: {}",
|
||||||
"details": details,
|
e
|
||||||
"uptime": std::time::SystemTime::now()
|
);
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
Response::builder()
|
||||||
.unwrap_or_default()
|
.status(StatusCode::INTERNAL_SERVER_ERROR)
|
||||||
.as_secs()
|
.body(Body::from("failed to build response"))
|
||||||
}))
|
.unwrap_or_else(|e| {
|
||||||
|
error!(
|
||||||
|
target: "rustfs::console::health",
|
||||||
|
"failed to build HEAD health empty response, reason: {}",
|
||||||
|
e
|
||||||
|
);
|
||||||
|
Response::new(Body::from(""))
|
||||||
|
})
|
||||||
|
}),
|
||||||
|
|
||||||
|
// Other methods: 405
|
||||||
|
_ => Response::builder()
|
||||||
|
.status(StatusCode::METHOD_NOT_ALLOWED)
|
||||||
|
.header("allow", "GET, HEAD")
|
||||||
|
.body(Body::from("Method Not Allowed"))
|
||||||
|
.unwrap_or_else(|e| {
|
||||||
|
error!(
|
||||||
|
target: "rustfs::console::health",
|
||||||
|
"failed to build 405 response: {}",
|
||||||
|
e
|
||||||
|
);
|
||||||
|
Response::new(Body::from("Method Not Allowed"))
|
||||||
|
}),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parse CORS allowed origins from configuration
|
/// Parse CORS allowed origins from configuration
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ use crate::auth::get_session_token;
|
|||||||
use crate::error::ApiError;
|
use crate::error::ApiError;
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use futures::{Stream, StreamExt};
|
use futures::{Stream, StreamExt};
|
||||||
use http::{HeaderMap, Uri};
|
use http::{HeaderMap, HeaderValue, Uri};
|
||||||
use hyper::StatusCode;
|
use hyper::StatusCode;
|
||||||
use matchit::Params;
|
use matchit::Params;
|
||||||
use rustfs_common::heal_channel::HealOpts;
|
use rustfs_common::heal_channel::HealOpts;
|
||||||
@@ -103,9 +103,23 @@ pub struct HealthCheckHandler {}
|
|||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
impl Operation for HealthCheckHandler {
|
impl Operation for HealthCheckHandler {
|
||||||
async fn call(&self, _req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
|
// Extract the original HTTP Method (encapsulated by s3s into S3Request)
|
||||||
|
let method = req.method;
|
||||||
|
|
||||||
|
// Only GET and HEAD are allowed
|
||||||
|
if method != http::Method::GET && method != http::Method::HEAD {
|
||||||
|
// 405 Method Not Allowed
|
||||||
|
let mut headers = HeaderMap::new();
|
||||||
|
headers.insert(http::header::ALLOW, HeaderValue::from_static("GET, HEAD"));
|
||||||
|
return Ok(S3Response::with_headers(
|
||||||
|
(StatusCode::METHOD_NOT_ALLOWED, Body::from("Method Not Allowed".to_string())),
|
||||||
|
headers,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
let health_info = json!({
|
let health_info = json!({
|
||||||
"status": "ok",
|
"status": "ok",
|
||||||
"service": "rustfs-endpoint",
|
"service": "rustfs-endpoint",
|
||||||
@@ -113,10 +127,19 @@ impl Operation for HealthCheckHandler {
|
|||||||
"version": env!("CARGO_PKG_VERSION")
|
"version": env!("CARGO_PKG_VERSION")
|
||||||
});
|
});
|
||||||
|
|
||||||
let body = serde_json::to_string(&health_info).unwrap_or_else(|_| "{}".to_string());
|
let mut headers = HeaderMap::new();
|
||||||
let response_body = Body::from(body);
|
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||||
|
|
||||||
Ok(S3Response::new((StatusCode::OK, response_body)))
|
if method == http::Method::HEAD {
|
||||||
|
// HEAD: only returns the header and status code, not the body
|
||||||
|
return Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), headers));
|
||||||
|
}
|
||||||
|
|
||||||
|
// GET: Return JSON body normally
|
||||||
|
let body_str = serde_json::to_string(&health_info).unwrap_or_else(|_| "{}".to_string());
|
||||||
|
let body = Body::from(body_str);
|
||||||
|
|
||||||
|
Ok(S3Response::with_headers((StatusCode::OK, body), headers))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -45,6 +45,7 @@ pub fn make_admin_route(console_enabled: bool) -> std::io::Result<impl S3Route>
|
|||||||
|
|
||||||
// Health check endpoint for monitoring and orchestration
|
// Health check endpoint for monitoring and orchestration
|
||||||
r.insert(Method::GET, "/health", AdminOperation(&HealthCheckHandler {}))?;
|
r.insert(Method::GET, "/health", AdminOperation(&HealthCheckHandler {}))?;
|
||||||
|
r.insert(Method::HEAD, "/health", AdminOperation(&HealthCheckHandler {}))?;
|
||||||
r.insert(Method::GET, "/profile/cpu", AdminOperation(&TriggerProfileCPU {}))?;
|
r.insert(Method::GET, "/profile/cpu", AdminOperation(&TriggerProfileCPU {}))?;
|
||||||
r.insert(Method::GET, "/profile/memory", AdminOperation(&TriggerProfileMemory {}))?;
|
r.insert(Method::GET, "/profile/memory", AdminOperation(&TriggerProfileMemory {}))?;
|
||||||
|
|
||||||
|
|||||||
@@ -85,7 +85,13 @@ where
|
|||||||
{
|
{
|
||||||
fn is_match(&self, method: &Method, uri: &Uri, headers: &HeaderMap, _: &mut Extensions) -> bool {
|
fn is_match(&self, method: &Method, uri: &Uri, headers: &HeaderMap, _: &mut Extensions) -> bool {
|
||||||
let path = uri.path();
|
let path = uri.path();
|
||||||
if method == Method::GET && (path == "/health" || path == "/profile/cpu" || path == "/profile/memory") {
|
// Profiling endpoints
|
||||||
|
if method == Method::GET && (path == "/profile/cpu" || path == "/profile/memory") {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Health check
|
||||||
|
if (method == Method::HEAD || method == Method::GET) && path == "/health" {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -105,9 +111,17 @@ where
|
|||||||
async fn check_access(&self, req: &mut S3Request<Body>) -> S3Result<()> {
|
async fn check_access(&self, req: &mut S3Request<Body>) -> S3Result<()> {
|
||||||
// Allow unauthenticated access to health check
|
// Allow unauthenticated access to health check
|
||||||
let path = req.uri.path();
|
let path = req.uri.path();
|
||||||
if req.method == Method::GET && (path == "/health" || path == "/profile/cpu" || path == "/profile/memory") {
|
|
||||||
|
// Profiling endpoints
|
||||||
|
if req.method == Method::GET && (path == "/profile/cpu" || path == "/profile/memory") {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Health check
|
||||||
|
if (req.method == Method::HEAD || req.method == Method::GET) && path == "/health" {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
// Allow unauthenticated access to console static files if console is enabled
|
// Allow unauthenticated access to console static files if console is enabled
|
||||||
if self.console_enabled && is_console_path(path) {
|
if self.console_enabled && is_console_path(path) {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
|
|||||||
Reference in New Issue
Block a user